Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore(tsdb): Initial commit of tsdb package 

* pulls in 1.x tsdb, compiles and passes test
pull/19446/head
Stuart Carnie 2020-04-22 13:19:20 -07:00
parent 0a9e8fdb4a
commit 92efddbfbe
No known key found for this signature in database
GPG Key ID: 848D9C9718D78B4F
434 changed files with 178666 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -92,6 +92,7 @@ require (
github.com/uber/jaeger-client-go v2.16.0+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/willf/bitset v1.1.9 // indirect
github.com/xlab/treeprint v1.0.0
github.com/yudai/gojsondiff v1.0.0
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
github.com/yudai/pp v2.0.1+incompatible // indirect

1531
influxql/query/call_iterator.go Normal file
View File

File diff suppressed because it is too large Load Diff

1213
influxql/query/call_iterator_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

88
influxql/query/cast.go Normal file
View File

@ -0,0 +1,88 @@
package query
import "github.com/influxdata/influxql"
// castToType will coerce the underlying interface type to another
// interface depending on the type.
func castToType(v interface{}, typ influxql.DataType) interface{} {
switch typ {
case influxql.Float:
if val, ok := castToFloat(v); ok {
v = val
}
case influxql.Integer:
if val, ok := castToInteger(v); ok {
v = val
}
case influxql.Unsigned:
if val, ok := castToUnsigned(v); ok {
v = val
}
case influxql.String, influxql.Tag:
if val, ok := castToString(v); ok {
v = val
}
case influxql.Boolean:
if val, ok := castToBoolean(v); ok {
v = val
}
}
return v
}
func castToFloat(v interface{}) (float64, bool) {
switch v := v.(type) {
case float64:
return v, true
case int64:
return float64(v), true
case uint64:
return float64(v), true
default:
return float64(0), false
}
}
func castToInteger(v interface{}) (int64, bool) {
switch v := v.(type) {
case float64:
return int64(v), true
case int64:
return v, true
case uint64:
return int64(v), true
default:
return int64(0), false
}
}
func castToUnsigned(v interface{}) (uint64, bool) {
switch v := v.(type) {
case float64:
return uint64(v), true
case uint64:
return v, true
case int64:
return uint64(v), true
default:
return uint64(0), false
}
}
func castToString(v interface{}) (string, bool) {
switch v := v.(type) {
case string:
return v, true
default:
return "", false
}
}
func castToBoolean(v interface{}) (bool, bool) {
switch v := v.(type) {
case bool:
return v, true
default:
return false, false
}
}

1205
influxql/query/compile.go Normal file
View File

File diff suppressed because it is too large Load Diff

438
influxql/query/compile_test.go Normal file
View File

@ -0,0 +1,438 @@
package query_test
import (
"testing"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxql"
)
func TestCompile_Success(t *testing.T) {
for _, tt := range []string{
`SELECT time, value FROM cpu`,
`SELECT value FROM cpu`,
`SELECT value, host FROM cpu`,
`SELECT * FROM cpu`,
`SELECT time, * FROM cpu`,
`SELECT value, * FROM cpu`,
`SELECT max(value) FROM cpu`,
`SELECT max(value), host FROM cpu`,
`SELECT max(value), * FROM cpu`,
`SELECT max(*) FROM cpu`,
`SELECT max(/val/) FROM cpu`,
`SELECT min(value) FROM cpu`,
`SELECT min(value), host FROM cpu`,
`SELECT min(value), * FROM cpu`,
`SELECT min(*) FROM cpu`,
`SELECT min(/val/) FROM cpu`,
`SELECT first(value) FROM cpu`,
`SELECT first(value), host FROM cpu`,
`SELECT first(value), * FROM cpu`,
`SELECT first(*) FROM cpu`,
`SELECT first(/val/) FROM cpu`,
`SELECT last(value) FROM cpu`,
`SELECT last(value), host FROM cpu`,
`SELECT last(value), * FROM cpu`,
`SELECT last(*) FROM cpu`,
`SELECT last(/val/) FROM cpu`,
`SELECT count(value) FROM cpu`,
`SELECT count(distinct(value)) FROM cpu`,
`SELECT count(distinct value) FROM cpu`,
`SELECT count(*) FROM cpu`,
`SELECT count(/val/) FROM cpu`,
`SELECT mean(value) FROM cpu`,
`SELECT mean(*) FROM cpu`,
`SELECT mean(/val/) FROM cpu`,
`SELECT min(value), max(value) FROM cpu`,
`SELECT min(*), max(*) FROM cpu`,
`SELECT min(/val/), max(/val/) FROM cpu`,
`SELECT first(value), last(value) FROM cpu`,
`SELECT first(*), last(*) FROM cpu`,
`SELECT first(/val/), last(/val/) FROM cpu`,
`SELECT count(value) FROM cpu WHERE time >= now() - 1h GROUP BY time(10m)`,
`SELECT distinct value FROM cpu`,
`SELECT distinct(value) FROM cpu`,
`SELECT value / total FROM cpu`,
`SELECT min(value) / total FROM cpu`,
`SELECT max(value) / total FROM cpu`,
`SELECT top(value, 1) FROM cpu`,
`SELECT top(value, host, 1) FROM cpu`,
`SELECT top(value, 1), host FROM cpu`,
`SELECT min(top) FROM (SELECT top(value, host, 1) FROM cpu) GROUP BY region`,
`SELECT bottom(value, 1) FROM cpu`,
`SELECT bottom(value, host, 1) FROM cpu`,
`SELECT bottom(value, 1), host FROM cpu`,
`SELECT max(bottom) FROM (SELECT bottom(value, host, 1) FROM cpu) GROUP BY region`,
`SELECT percentile(value, 75) FROM cpu`,
`SELECT percentile(value, 75.0) FROM cpu`,
`SELECT sample(value, 2) FROM cpu`,
`SELECT sample(*, 2) FROM cpu`,
`SELECT sample(/val/, 2) FROM cpu`,
`SELECT elapsed(value) FROM cpu`,
`SELECT elapsed(value, 10s) FROM cpu`,
`SELECT integral(value) FROM cpu`,
`SELECT integral(value, 10s) FROM cpu`,
`SELECT max(value) FROM cpu WHERE time >= now() - 1m GROUP BY time(10s, 5s)`,
`SELECT max(value) FROM cpu WHERE time >= now() - 1m GROUP BY time(10s, '2000-01-01T00:00:05Z')`,
`SELECT max(value) FROM cpu WHERE time >= now() - 1m GROUP BY time(10s, now())`,
`SELECT max(mean) FROM (SELECT mean(value) FROM cpu GROUP BY host)`,
`SELECT max(derivative) FROM (SELECT derivative(mean(value)) FROM cpu) WHERE time >= now() - 1m GROUP BY time(10s)`,
`SELECT max(value) FROM (SELECT value + total FROM cpu) WHERE time >= now() - 1m GROUP BY time(10s)`,
`SELECT value FROM cpu WHERE time >= '2000-01-01T00:00:00Z' AND time <= '2000-01-01T01:00:00Z'`,
`SELECT value FROM (SELECT value FROM cpu) ORDER BY time DESC`,
`SELECT count(distinct(value)), max(value) FROM cpu`,
`SELECT derivative(distinct(value)), difference(distinct(value)) FROM cpu WHERE time >= now() - 1m GROUP BY time(5s)`,
`SELECT moving_average(distinct(value), 3) FROM cpu WHERE time >= now() - 5m GROUP BY time(1m)`,
`SELECT elapsed(distinct(value)) FROM cpu WHERE time >= now() - 5m GROUP BY time(1m)`,
`SELECT cumulative_sum(distinct(value)) FROM cpu WHERE time >= now() - 5m GROUP BY time(1m)`,
`SELECT last(value) / (1 - 0) FROM cpu`,
`SELECT abs(value) FROM cpu`,
`SELECT sin(value) FROM cpu`,
`SELECT cos(value) FROM cpu`,
`SELECT tan(value) FROM cpu`,
`SELECT asin(value) FROM cpu`,
`SELECT acos(value) FROM cpu`,
`SELECT atan(value) FROM cpu`,
`SELECT sqrt(value) FROM cpu`,
`SELECT pow(value, 2) FROM cpu`,
`SELECT pow(value, 3.14) FROM cpu`,
`SELECT pow(2, value) FROM cpu`,
`SELECT pow(3.14, value) FROM cpu`,
`SELECT exp(value) FROM cpu`,
`SELECT atan2(value, 0.1) FROM cpu`,
`SELECT atan2(0.2, value) FROM cpu`,
`SELECT atan2(value, 1) FROM cpu`,
`SELECT atan2(2, value) FROM cpu`,
`SELECT ln(value) FROM cpu`,
`SELECT log(value, 2) FROM cpu`,
`SELECT log2(value) FROM cpu`,
`SELECT log10(value) FROM cpu`,
`SELECT sin(value) - sin(1.3) FROM cpu`,
`SELECT value FROM cpu WHERE sin(value) > 0.5`,
`SELECT sum("out")/sum("in") FROM (SELECT derivative("out") AS "out", derivative("in") AS "in" FROM "m0" WHERE time >= now() - 5m GROUP BY "index") GROUP BY time(1m) fill(none)`,
} {
t.Run(tt, func(t *testing.T) {
stmt, err := influxql.ParseStatement(tt)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
s := stmt.(*influxql.SelectStatement)
opt := query.CompileOptions{}
if _, err := query.Compile(s, opt); err != nil {
t.Errorf("unexpected error: %s", err)
}
})
}
}
func TestCompile_Failures(t *testing.T) {
for _, tt := range []struct {
s string
err string
}{
{s: `SELECT time FROM cpu`, err: `at least 1 non-time field must be queried`},
{s: `SELECT value, mean(value) FROM cpu`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT value, max(value), min(value) FROM cpu`, err: `mixing multiple selector functions with tags or fields is not supported`},
{s: `SELECT top(value, 10), max(value) FROM cpu`, err: `selector function top() cannot be combined with other functions`},
{s: `SELECT bottom(value, 10), max(value) FROM cpu`, err: `selector function bottom() cannot be combined with other functions`},
{s: `SELECT count() FROM cpu`, err: `invalid number of arguments for count, expected 1, got 0`},
{s: `SELECT count(value, host) FROM cpu`, err: `invalid number of arguments for count, expected 1, got 2`},
{s: `SELECT min() FROM cpu`, err: `invalid number of arguments for min, expected 1, got 0`},
{s: `SELECT min(value, host) FROM cpu`, err: `invalid number of arguments for min, expected 1, got 2`},
{s: `SELECT max() FROM cpu`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT max(value, host) FROM cpu`, err: `invalid number of arguments for max, expected 1, got 2`},
{s: `SELECT sum() FROM cpu`, err: `invalid number of arguments for sum, expected 1, got 0`},
{s: `SELECT sum(value, host) FROM cpu`, err: `invalid number of arguments for sum, expected 1, got 2`},
{s: `SELECT first() FROM cpu`, err: `invalid number of arguments for first, expected 1, got 0`},
{s: `SELECT first(value, host) FROM cpu`, err: `invalid number of arguments for first, expected 1, got 2`},
{s: `SELECT last() FROM cpu`, err: `invalid number of arguments for last, expected 1, got 0`},
{s: `SELECT last(value, host) FROM cpu`, err: `invalid number of arguments for last, expected 1, got 2`},
{s: `SELECT mean() FROM cpu`, err: `invalid number of arguments for mean, expected 1, got 0`},
{s: `SELECT mean(value, host) FROM cpu`, err: `invalid number of arguments for mean, expected 1, got 2`},
{s: `SELECT distinct(value), max(value) FROM cpu`, err: `aggregate function distinct() cannot be combined with other functions or fields`},
{s: `SELECT count(distinct()) FROM cpu`, err: `distinct function requires at least one argument`},
{s: `SELECT count(distinct(value, host)) FROM cpu`, err: `distinct function can only have one argument`},
{s: `SELECT count(distinct(2)) FROM cpu`, err: `expected field argument in distinct()`},
{s: `SELECT value FROM cpu GROUP BY now()`, err: `only time() calls allowed in dimensions`},
{s: `SELECT value FROM cpu GROUP BY time()`, err: `time dimension expected 1 or 2 arguments`},
{s: `SELECT value FROM cpu GROUP BY time(5m, 30s, 1ms)`, err: `time dimension expected 1 or 2 arguments`},
{s: `SELECT value FROM cpu GROUP BY time('unexpected')`, err: `time dimension must have duration argument`},
{s: `SELECT value FROM cpu GROUP BY time(5m), time(1m)`, err: `multiple time dimensions not allowed`},
{s: `SELECT value FROM cpu GROUP BY time(5m, unexpected())`, err: `time dimension offset function must be now()`},
{s: `SELECT value FROM cpu GROUP BY time(5m, now(1m))`, err: `time dimension offset now() function requires no arguments`},
{s: `SELECT value FROM cpu GROUP BY time(5m, 'unexpected')`, err: `time dimension offset must be duration or now()`},
{s: `SELECT value FROM cpu GROUP BY 'unexpected'`, err: `only time and tag dimensions allowed`},
{s: `SELECT top(value) FROM cpu`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT top('unexpected', 5) FROM cpu`, err: `expected first argument to be a field in top(), found 'unexpected'`},
{s: `SELECT top(value, 'unexpected', 5) FROM cpu`, err: `only fields or tags are allowed in top(), found 'unexpected'`},
{s: `SELECT top(value, 2.5) FROM cpu`, err: `expected integer as last argument in top(), found 2.500`},
{s: `SELECT top(value, -1) FROM cpu`, err: `limit (-1) in top function must be at least 1`},
{s: `SELECT top(value, 3) FROM cpu LIMIT 2`, err: `limit (3) in top function can not be larger than the LIMIT (2) in the select statement`},
{s: `SELECT bottom(value) FROM cpu`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT bottom('unexpected', 5) FROM cpu`, err: `expected first argument to be a field in bottom(), found 'unexpected'`},
{s: `SELECT bottom(value, 'unexpected', 5) FROM cpu`, err: `only fields or tags are allowed in bottom(), found 'unexpected'`},
{s: `SELECT bottom(value, 2.5) FROM cpu`, err: `expected integer as last argument in bottom(), found 2.500`},
{s: `SELECT bottom(value, -1) FROM cpu`, err: `limit (-1) in bottom function must be at least 1`},
{s: `SELECT bottom(value, 3) FROM cpu LIMIT 2`, err: `limit (3) in bottom function can not be larger than the LIMIT (2) in the select statement`},
// TODO(jsternberg): This query is wrong, but we cannot enforce this because of previous behavior: https://github.com/influxdata/influxdb/pull/8771
//{s: `SELECT value FROM cpu WHERE time >= now() - 10m OR time < now() - 5m`, err: `cannot use OR with time conditions`},
{s: `SELECT value FROM cpu WHERE value`, err: `invalid condition expression: value`},
{s: `SELECT count(value), * FROM cpu`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT max(*), host FROM cpu`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT count(value), /ho/ FROM cpu`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT max(/val/), * FROM cpu`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT a(value) FROM cpu`, err: `undefined function a()`},
{s: `SELECT count(max(value)) FROM myseries`, err: `expected field argument in count()`},
{s: `SELECT count(distinct('value')) FROM myseries`, err: `expected field argument in distinct()`},
{s: `SELECT distinct('value') FROM myseries`, err: `expected field argument in distinct()`},
{s: `SELECT min(max(value)) FROM myseries`, err: `expected field argument in min()`},
{s: `SELECT min(distinct(value)) FROM myseries`, err: `expected field argument in min()`},
{s: `SELECT max(max(value)) FROM myseries`, err: `expected field argument in max()`},
{s: `SELECT sum(max(value)) FROM myseries`, err: `expected field argument in sum()`},
{s: `SELECT first(max(value)) FROM myseries`, err: `expected field argument in first()`},
{s: `SELECT last(max(value)) FROM myseries`, err: `expected field argument in last()`},
{s: `SELECT mean(max(value)) FROM myseries`, err: `expected field argument in mean()`},
{s: `SELECT median(max(value)) FROM myseries`, err: `expected field argument in median()`},
{s: `SELECT mode(max(value)) FROM myseries`, err: `expected field argument in mode()`},
{s: `SELECT stddev(max(value)) FROM myseries`, err: `expected field argument in stddev()`},
{s: `SELECT spread(max(value)) FROM myseries`, err: `expected field argument in spread()`},
{s: `SELECT top() FROM myseries`, err: `invalid number of arguments for top, expected at least 2, got 0`},
{s: `SELECT top(field1) FROM myseries`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT top(field1,foo) FROM myseries`, err: `expected integer as last argument in top(), found foo`},
{s: `SELECT top(field1,host,'server',foo) FROM myseries`, err: `expected integer as last argument in top(), found foo`},
{s: `SELECT top(field1,5,'server',2) FROM myseries`, err: `only fields or tags are allowed in top(), found 5`},
{s: `SELECT top(field1,max(foo),'server',2) FROM myseries`, err: `only fields or tags are allowed in top(), found max(foo)`},
{s: `SELECT top(value, 10) + count(value) FROM myseries`, err: `selector function top() cannot be combined with other functions`},
{s: `SELECT top(max(value), 10) FROM myseries`, err: `expected first argument to be a field in top(), found max(value)`},
{s: `SELECT bottom() FROM myseries`, err: `invalid number of arguments for bottom, expected at least 2, got 0`},
{s: `SELECT bottom(field1) FROM myseries`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT bottom(field1,foo) FROM myseries`, err: `expected integer as last argument in bottom(), found foo`},
{s: `SELECT bottom(field1,host,'server',foo) FROM myseries`, err: `expected integer as last argument in bottom(), found foo`},
{s: `SELECT bottom(field1,5,'server',2) FROM myseries`, err: `only fields or tags are allowed in bottom(), found 5`},
{s: `SELECT bottom(field1,max(foo),'server',2) FROM myseries`, err: `only fields or tags are allowed in bottom(), found max(foo)`},
{s: `SELECT bottom(value, 10) + count(value) FROM myseries`, err: `selector function bottom() cannot be combined with other functions`},
{s: `SELECT bottom(max(value), 10) FROM myseries`, err: `expected first argument to be a field in bottom(), found max(value)`},
{s: `SELECT top(value, 10), bottom(value, 10) FROM cpu`, err: `selector function top() cannot be combined with other functions`},
{s: `SELECT bottom(value, 10), top(value, 10) FROM cpu`, err: `selector function bottom() cannot be combined with other functions`},
{s: `SELECT sample(value) FROM myseries`, err: `invalid number of arguments for sample, expected 2, got 1`},
{s: `SELECT sample(value, 2, 3) FROM myseries`, err: `invalid number of arguments for sample, expected 2, got 3`},
{s: `SELECT sample(value, 0) FROM myseries`, err: `sample window must be greater than 1, got 0`},
{s: `SELECT sample(value, 2.5) FROM myseries`, err: `expected integer argument in sample()`},
{s: `SELECT percentile() FROM myseries`, err: `invalid number of arguments for percentile, expected 2, got 0`},
{s: `SELECT percentile(field1) FROM myseries`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT percentile(field1, foo) FROM myseries`, err: `expected float argument in percentile()`},
{s: `SELECT percentile(max(field1), 75) FROM myseries`, err: `expected field argument in percentile()`},
{s: `SELECT field1 FROM foo group by time(1s)`, err: `GROUP BY requires at least one aggregate function`},
{s: `SELECT field1 FROM foo fill(none)`, err: `fill(none) must be used with a function`},
{s: `SELECT field1 FROM foo fill(linear)`, err: `fill(linear) must be used with a function`},
{s: `SELECT count(value), value FROM foo`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT count(value) FROM foo group by time`, err: `time() is a function and expects at least one argument`},
{s: `SELECT count(value) FROM foo group by 'time'`, err: `only time and tag dimensions allowed`},
{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time()`, err: `time dimension expected 1 or 2 arguments`},
{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(b)`, err: `time dimension must have duration argument`},
{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s), time(2s)`, err: `multiple time dimensions not allowed`},
{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s, b)`, err: `time dimension offset must be duration or now()`},
{s: `SELECT count(value) FROM foo where time > now() and time < now() group by time(1s, '5s')`, err: `time dimension offset must be duration or now()`},
{s: `SELECT distinct(field1), sum(field1) FROM myseries`, err: `aggregate function distinct() cannot be combined with other functions or fields`},
{s: `SELECT distinct(field1), field2 FROM myseries`, err: `aggregate function distinct() cannot be combined with other functions or fields`},
{s: `SELECT distinct(field1, field2) FROM myseries`, err: `distinct function can only have one argument`},
{s: `SELECT distinct() FROM myseries`, err: `distinct function requires at least one argument`},
{s: `SELECT distinct field1, field2 FROM myseries`, err: `aggregate function distinct() cannot be combined with other functions or fields`},
{s: `SELECT count(distinct field1, field2) FROM myseries`, err: `invalid number of arguments for count, expected 1, got 2`},
{s: `select count(distinct(too, many, arguments)) from myseries`, err: `distinct function can only have one argument`},
{s: `select count() from myseries`, err: `invalid number of arguments for count, expected 1, got 0`},
{s: `SELECT derivative(field1), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `select derivative() from myseries`, err: `invalid number of arguments for derivative, expected at least 1 but no more than 2, got 0`},
{s: `select derivative(mean(value), 1h, 3) from myseries`, err: `invalid number of arguments for derivative, expected at least 1 but no more than 2, got 3`},
{s: `SELECT derivative(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to derivative`},
{s: `SELECT derivative(top(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT derivative(bottom(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT derivative(max()) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT derivative(percentile(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT derivative(mean(value), 1h) FROM myseries where time < now() and time > now() - 1d`, err: `derivative aggregate requires a GROUP BY interval`},
{s: `SELECT derivative(value, -2h) FROM myseries`, err: `duration argument must be positive, got -2h`},
{s: `SELECT derivative(value, 10) FROM myseries`, err: `second argument to derivative must be a duration, got *influxql.IntegerLiteral`},
{s: `SELECT derivative(f, true) FROM myseries`, err: `second argument to derivative must be a duration, got *influxql.BooleanLiteral`},
{s: `SELECT non_negative_derivative(field1), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `select non_negative_derivative() from myseries`, err: `invalid number of arguments for non_negative_derivative, expected at least 1 but no more than 2, got 0`},
{s: `select non_negative_derivative(mean(value), 1h, 3) from myseries`, err: `invalid number of arguments for non_negative_derivative, expected at least 1 but no more than 2, got 3`},
{s: `SELECT non_negative_derivative(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to non_negative_derivative`},
{s: `SELECT non_negative_derivative(top(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT non_negative_derivative(bottom(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT non_negative_derivative(max()) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT non_negative_derivative(mean(value), 1h) FROM myseries where time < now() and time > now() - 1d`, err: `non_negative_derivative aggregate requires a GROUP BY interval`},
{s: `SELECT non_negative_derivative(percentile(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT non_negative_derivative(value, -2h) FROM myseries`, err: `duration argument must be positive, got -2h`},
{s: `SELECT non_negative_derivative(value, 10) FROM myseries`, err: `second argument to non_negative_derivative must be a duration, got *influxql.IntegerLiteral`},
{s: `SELECT difference(field1), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT difference() from myseries`, err: `invalid number of arguments for difference, expected 1, got 0`},
{s: `SELECT difference(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to difference`},
{s: `SELECT difference(top(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT difference(bottom(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT difference(max()) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT difference(percentile(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT difference(mean(value)) FROM myseries where time < now() and time > now() - 1d`, err: `difference aggregate requires a GROUP BY interval`},
{s: `SELECT non_negative_difference(field1), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT non_negative_difference() from myseries`, err: `invalid number of arguments for non_negative_difference, expected 1, got 0`},
{s: `SELECT non_negative_difference(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to non_negative_difference`},
{s: `SELECT non_negative_difference(top(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT non_negative_difference(bottom(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT non_negative_difference(max()) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT non_negative_difference(percentile(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT non_negative_difference(mean(value)) FROM myseries where time < now() and time > now() - 1d`, err: `non_negative_difference aggregate requires a GROUP BY interval`},
{s: `SELECT elapsed() FROM myseries`, err: `invalid number of arguments for elapsed, expected at least 1 but no more than 2, got 0`},
{s: `SELECT elapsed(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to elapsed`},
{s: `SELECT elapsed(value, 1s, host) FROM myseries`, err: `invalid number of arguments for elapsed, expected at least 1 but no more than 2, got 3`},
{s: `SELECT elapsed(value, 0s) FROM myseries`, err: `duration argument must be positive, got 0s`},
{s: `SELECT elapsed(value, -10s) FROM myseries`, err: `duration argument must be positive, got -10s`},
{s: `SELECT elapsed(value, 10) FROM myseries`, err: `second argument to elapsed must be a duration, got *influxql.IntegerLiteral`},
{s: `SELECT elapsed(top(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT elapsed(bottom(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT elapsed(max()) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT elapsed(percentile(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT elapsed(mean(value)) FROM myseries where time < now() and time > now() - 1d`, err: `elapsed aggregate requires a GROUP BY interval`},
{s: `SELECT moving_average(field1, 2), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT moving_average(field1, 1), field1 FROM myseries`, err: `moving_average window must be greater than 1, got 1`},
{s: `SELECT moving_average(field1, 0), field1 FROM myseries`, err: `moving_average window must be greater than 1, got 0`},
{s: `SELECT moving_average(field1, -1), field1 FROM myseries`, err: `moving_average window must be greater than 1, got -1`},
{s: `SELECT moving_average(field1, 2.0), field1 FROM myseries`, err: `second argument for moving_average must be an integer, got *influxql.NumberLiteral`},
{s: `SELECT moving_average() from myseries`, err: `invalid number of arguments for moving_average, expected 2, got 0`},
{s: `SELECT moving_average(value) FROM myseries`, err: `invalid number of arguments for moving_average, expected 2, got 1`},
{s: `SELECT moving_average(value, 2) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to moving_average`},
{s: `SELECT moving_average(top(value), 2) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT moving_average(bottom(value), 2) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT moving_average(max(), 2) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT moving_average(percentile(value), 2) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT moving_average(mean(value), 2) FROM myseries where time < now() and time > now() - 1d`, err: `moving_average aggregate requires a GROUP BY interval`},
{s: `SELECT cumulative_sum(field1), field1 FROM myseries`, err: `mixing aggregate and non-aggregate queries is not supported`},
{s: `SELECT cumulative_sum() from myseries`, err: `invalid number of arguments for cumulative_sum, expected 1, got 0`},
{s: `SELECT cumulative_sum(value) FROM myseries group by time(1h)`, err: `aggregate function required inside the call to cumulative_sum`},
{s: `SELECT cumulative_sum(top(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for top, expected at least 2, got 1`},
{s: `SELECT cumulative_sum(bottom(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for bottom, expected at least 2, got 1`},
{s: `SELECT cumulative_sum(max()) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for max, expected 1, got 0`},
{s: `SELECT cumulative_sum(percentile(value)) FROM myseries where time < now() and time > now() - 1d group by time(1h)`, err: `invalid number of arguments for percentile, expected 2, got 1`},
{s: `SELECT cumulative_sum(mean(value)) FROM myseries where time < now() and time > now() - 1d`, err: `cumulative_sum aggregate requires a GROUP BY interval`},
{s: `SELECT integral() FROM myseries`, err: `invalid number of arguments for integral, expected at least 1 but no more than 2, got 0`},
{s: `SELECT integral(value, 10s, host) FROM myseries`, err: `invalid number of arguments for integral, expected at least 1 but no more than 2, got 3`},
{s: `SELECT integral(value, -10s) FROM myseries`, err: `duration argument must be positive, got -10s`},
{s: `SELECT integral(value, 10) FROM myseries`, err: `second argument must be a duration`},
{s: `SELECT holt_winters(value) FROM myseries where time < now() and time > now() - 1d`, err: `invalid number of arguments for holt_winters, expected 3, got 1`},
{s: `SELECT holt_winters(value, 10, 2) FROM myseries where time < now() and time > now() - 1d`, err: `must use aggregate function with holt_winters`},
{s: `SELECT holt_winters(min(value), 10, 2) FROM myseries where time < now() and time > now() - 1d`, err: `holt_winters aggregate requires a GROUP BY interval`},
{s: `SELECT holt_winters(min(value), 0, 2) FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `second arg to holt_winters must be greater than 0, got 0`},
{s: `SELECT holt_winters(min(value), false, 2) FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `expected integer argument as second arg in holt_winters`},
{s: `SELECT holt_winters(min(value), 10, 'string') FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `expected integer argument as third arg in holt_winters`},
{s: `SELECT holt_winters(min(value), 10, -1) FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `third arg to holt_winters cannot be negative, got -1`},
{s: `SELECT holt_winters_with_fit(value) FROM myseries where time < now() and time > now() - 1d`, err: `invalid number of arguments for holt_winters_with_fit, expected 3, got 1`},
{s: `SELECT holt_winters_with_fit(value, 10, 2) FROM myseries where time < now() and time > now() - 1d`, err: `must use aggregate function with holt_winters_with_fit`},
{s: `SELECT holt_winters_with_fit(min(value), 10, 2) FROM myseries where time < now() and time > now() - 1d`, err: `holt_winters_with_fit aggregate requires a GROUP BY interval`},
{s: `SELECT holt_winters_with_fit(min(value), 0, 2) FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `second arg to holt_winters_with_fit must be greater than 0, got 0`},
{s: `SELECT holt_winters_with_fit(min(value), false, 2) FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `expected integer argument as second arg in holt_winters_with_fit`},
{s: `SELECT holt_winters_with_fit(min(value), 10, 'string') FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `expected integer argument as third arg in holt_winters_with_fit`},
{s: `SELECT holt_winters_with_fit(min(value), 10, -1) FROM myseries where time < now() and time > now() - 1d GROUP BY time(1d)`, err: `third arg to holt_winters_with_fit cannot be negative, got -1`},
{s: `SELECT mean(value) + value FROM cpu WHERE time < now() and time > now() - 1h GROUP BY time(10m)`, err: `mixing aggregate and non-aggregate queries is not supported`},
// TODO: Remove this restriction in the future: https://github.com/influxdata/influxdb/issues/5968
{s: `SELECT mean(cpu_total - cpu_idle) FROM cpu`, err: `expected field argument in mean()`},
{s: `SELECT derivative(mean(cpu_total - cpu_idle), 1s) FROM cpu WHERE time < now() AND time > now() - 1d GROUP BY time(1h)`, err: `expected field argument in mean()`},
// TODO: The error message will change when math is allowed inside an aggregate: https://github.com/influxdata/influxdb/pull/5990#issuecomment-195565870
{s: `SELECT count(foo + sum(bar)) FROM cpu`, err: `expected field argument in count()`},
{s: `SELECT (count(foo + sum(bar))) FROM cpu`, err: `expected field argument in count()`},
{s: `SELECT sum(value) + count(foo + sum(bar)) FROM cpu`, err: `expected field argument in count()`},
{s: `SELECT top(value, 2), max(value) FROM cpu`, err: `selector function top() cannot be combined with other functions`},
{s: `SELECT bottom(value, 2), max(value) FROM cpu`, err: `selector function bottom() cannot be combined with other functions`},
{s: `SELECT min(derivative) FROM (SELECT derivative(mean(value), 1h) FROM myseries) where time < now() and time > now() - 1d`, err: `derivative aggregate requires a GROUP BY interval`},
{s: `SELECT min(mean) FROM (SELECT mean(value) FROM myseries GROUP BY time)`, err: `time() is a function and expects at least one argument`},
{s: `SELECT value FROM myseries WHERE value OR time >= now() - 1m`, err: `invalid condition expression: value`},
{s: `SELECT value FROM myseries WHERE time >= now() - 1m OR value`, err: `invalid condition expression: value`},
{s: `SELECT value FROM (SELECT value FROM cpu ORDER BY time DESC) ORDER BY time ASC`, err: `subqueries must be ordered in the same direction as the query itself`},
{s: `SELECT sin(value, 3) FROM cpu`, err: `invalid number of arguments for sin, expected 1, got 2`},
{s: `SELECT cos(2.3, value, 3) FROM cpu`, err: `invalid number of arguments for cos, expected 1, got 3`},
{s: `SELECT tan(value, 3) FROM cpu`, err: `invalid number of arguments for tan, expected 1, got 2`},
{s: `SELECT asin(value, 3) FROM cpu`, err: `invalid number of arguments for asin, expected 1, got 2`},
{s: `SELECT acos(value, 3.2) FROM cpu`, err: `invalid number of arguments for acos, expected 1, got 2`},
{s: `SELECT atan() FROM cpu`, err: `invalid number of arguments for atan, expected 1, got 0`},
{s: `SELECT sqrt(42, 3, 4) FROM cpu`, err: `invalid number of arguments for sqrt, expected 1, got 3`},
{s: `SELECT abs(value, 3) FROM cpu`, err: `invalid number of arguments for abs, expected 1, got 2`},
{s: `SELECT ln(value, 3) FROM cpu`, err: `invalid number of arguments for ln, expected 1, got 2`},
{s: `SELECT log2(value, 3) FROM cpu`, err: `invalid number of arguments for log2, expected 1, got 2`},
{s: `SELECT log10(value, 3) FROM cpu`, err: `invalid number of arguments for log10, expected 1, got 2`},
{s: `SELECT pow(value, 3, 3) FROM cpu`, err: `invalid number of arguments for pow, expected 2, got 3`},
{s: `SELECT atan2(value, 3, 3) FROM cpu`, err: `invalid number of arguments for atan2, expected 2, got 3`},
{s: `SELECT sin(1.3) FROM cpu`, err: `field must contain at least one variable`},
{s: `SELECT nofunc(1.3) FROM cpu`, err: `undefined function nofunc()`},
} {
t.Run(tt.s, func(t *testing.T) {
stmt, err := influxql.ParseStatement(tt.s)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
s := stmt.(*influxql.SelectStatement)
opt := query.CompileOptions{}
if _, err := query.Compile(s, opt); err == nil {
t.Error("expected error")
} else if have, want := err.Error(), tt.err; have != want {
t.Errorf("unexpected error: %s != %s", have, want)
}
})
}
}
func TestPrepare_MapShardsTimeRange(t *testing.T) {
for _, tt := range []struct {
s string
start, end string
}{
{
s: `SELECT max(value) FROM cpu WHERE time >= '2018-09-03T15:00:00Z' AND time <= '2018-09-03T16:00:00Z' GROUP BY time(10m)`,
start: "2018-09-03T15:00:00Z",
end: "2018-09-03T16:00:00Z",
},
{
s: `SELECT derivative(mean(value)) FROM cpu WHERE time >= '2018-09-03T15:00:00Z' AND time <= '2018-09-03T16:00:00Z' GROUP BY time(10m)`,
start: "2018-09-03T14:50:00Z",
end: "2018-09-03T16:00:00Z",
},
{
s: `SELECT moving_average(mean(value), 3) FROM cpu WHERE time >= '2018-09-03T15:00:00Z' AND time <= '2018-09-03T16:00:00Z' GROUP BY time(10m)`,
start: "2018-09-03T14:30:00Z",
end: "2018-09-03T16:00:00Z",
},
{
s: `SELECT moving_average(mean(value), 3) FROM cpu WHERE time <= '2018-09-03T16:00:00Z' GROUP BY time(10m)`,
start: "1677-09-21T00:12:43.145224194Z",
end: "2018-09-03T16:00:00Z",
},
} {
t.Run(tt.s, func(t *testing.T) {
stmt, err := influxql.ParseStatement(tt.s)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
s := stmt.(*influxql.SelectStatement)
opt := query.CompileOptions{}
c, err := query.Compile(s, opt)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
shardMapper := ShardMapper{
MapShardsFn: func(_ influxql.Sources, tr influxql.TimeRange) query.ShardGroup {
if got, want := tr.Min, mustParseTime(tt.start); !got.Equal(want) {
t.Errorf("unexpected start time: got=%s want=%s", got, want)
}
if got, want := tr.Max, mustParseTime(tt.end); !got.Equal(want) {
t.Errorf("unexpected end time: got=%s want=%s", got, want)
}
return &ShardGroup{}
},
}
if _, err := c.Prepare(&shardMapper, query.SelectOptions{}); err != nil {
t.Fatalf("unexpected error: %s", err)
}
})
}
}

447
influxql/query/cursor.go Normal file
View File

@ -0,0 +1,447 @@
package query
import (
"math"
"time"
"github.com/influxdata/influxql"
)
var NullFloat interface{} = (*float64)(nil)
// Series represents the metadata about a series.
type Series struct {
// Name is the measurement name.
Name string
// Tags for the series.
Tags Tags
// This is an internal id used to easily compare if a series is the
// same as another series. Whenever the internal cursor changes
// to a new series, this id gets incremented. It is not exposed to
// the user so we can implement this in whatever way we want.
// If a series is not generated by a cursor, this id is zero and
// it will instead attempt to compare the name and tags.
id uint64
}
// SameSeries checks if this is the same series as another one.
// It does not necessarily check for equality so this is different from
// checking to see if the name and tags are the same. It checks whether
// the two are part of the same series in the response.
func (s Series) SameSeries(other Series) bool {
if s.id != 0 && other.id != 0 {
return s.id == other.id
}
return s.Name == other.Name && s.Tags.ID() == other.Tags.ID()
}
// Equal checks to see if the Series are identical.
func (s Series) Equal(other Series) bool {
if s.id != 0 && other.id != 0 {
// If the ids are the same, then we can short-circuit and assume they
// are the same. If they are not the same, do the long check since
// they may still be identical, but not necessarily generated from
// the same cursor.
if s.id == other.id {
return true
}
}
return s.Name == other.Name && s.Tags.ID() == other.Tags.ID()
}
// Row represents a single row returned by the query engine.
type Row struct {
// Time returns the time for this row. If the cursor was created to
// return time as one of the values, the time will also be included as
// a time.Time in the appropriate column within Values.
// This ensures that time is always present in the Row structure
// even if it hasn't been requested in the output.
Time int64
// Series contains the series metadata for this row.
Series Series
// Values contains the values within the current row.
Values []interface{}
}
type Cursor interface {
// Scan will retrieve the next row and assign the result to
// the passed in Row. If the Row has not been initialized, the Cursor
// will initialize the Row.
// To increase speed and memory usage, the same Row can be used and
// the previous values will be overwritten while using the same memory.
Scan(row *Row) bool
// Stats returns the IteratorStats from the underlying iterators.
Stats() IteratorStats
// Err returns any errors that were encountered from scanning the rows.
Err() error
// Columns returns the column names and types.
Columns() []influxql.VarRef
// Close closes the underlying resources that the cursor is using.
Close() error
}
// RowCursor returns a Cursor that iterates over Rows.
func RowCursor(rows []Row, columns []influxql.VarRef) Cursor {
return &rowCursor{
rows: rows,
columns: columns,
}
}
type rowCursor struct {
rows []Row
columns []influxql.VarRef
series Series
}
func (cur *rowCursor) Scan(row *Row) bool {
if len(cur.rows) == 0 {
return false
}
*row = cur.rows[0]
if row.Series.Name != cur.series.Name || !row.Series.Tags.Equals(&cur.series.Tags) {
cur.series.Name = row.Series.Name
cur.series.Tags = row.Series.Tags
cur.series.id++
}
cur.rows = cur.rows[1:]
return true
}
func (cur *rowCursor) Stats() IteratorStats {
return IteratorStats{}
}
func (cur *rowCursor) Err() error {
return nil
}
func (cur *rowCursor) Columns() []influxql.VarRef {
return cur.columns
}
func (cur *rowCursor) Close() error {
return nil
}
type scannerFunc func(m map[string]interface{}) (int64, string, Tags)
type scannerCursorBase struct {
fields []influxql.Expr
m map[string]interface{}
series Series
columns []influxql.VarRef
loc *time.Location
scan scannerFunc
valuer influxql.ValuerEval
}
func newScannerCursorBase(scan scannerFunc, fields []*influxql.Field, loc *time.Location) scannerCursorBase {
typmap := FunctionTypeMapper{}
exprs := make([]influxql.Expr, len(fields))
columns := make([]influxql.VarRef, len(fields))
for i, f := range fields {
exprs[i] = f.Expr
columns[i] = influxql.VarRef{
Val: f.Name(),
Type: influxql.EvalType(f.Expr, nil, typmap),
}
}
if loc == nil {
loc = time.UTC
}
m := make(map[string]interface{})
return scannerCursorBase{
fields: exprs,
m: m,
columns: columns,
loc: loc,
scan: scan,
valuer: influxql.ValuerEval{
Valuer: influxql.MultiValuer(
MathValuer{},
influxql.MapValuer(m),
),
IntegerFloatDivision: true,
},
}
}
func (cur *scannerCursorBase) Scan(row *Row) bool {
ts, name, tags := cur.scan(cur.m)
if ts == ZeroTime {
return false
}
row.Time = ts
if name != cur.series.Name || tags.ID() != cur.series.Tags.ID() {
cur.series.Name = name
cur.series.Tags = tags
cur.series.id++
}
row.Series = cur.series
if len(cur.columns) > len(row.Values) {
row.Values = make([]interface{}, len(cur.columns))
}
for i, expr := range cur.fields {
// A special case if the field is time to reduce memory allocations.
if ref, ok := expr.(*influxql.VarRef); ok && ref.Val == "time" {
row.Values[i] = time.Unix(0, row.Time).In(cur.loc)
continue
}
v := cur.valuer.Eval(expr)
if fv, ok := v.(float64); ok && math.IsNaN(fv) {
// If the float value is NaN, convert it to a null float
// so this can be serialized correctly, but not mistaken for
// a null value that needs to be filled.
v = NullFloat
}
row.Values[i] = v
}
return true
}
func (cur *scannerCursorBase) Columns() []influxql.VarRef {
return cur.columns
}
func (cur *scannerCursorBase) clear(m map[string]interface{}) {
for k := range m {
delete(m, k)
}
}
var _ Cursor = (*scannerCursor)(nil)
type scannerCursor struct {
scanner IteratorScanner
scannerCursorBase
}
func newScannerCursor(s IteratorScanner, fields []*influxql.Field, opt IteratorOptions) *scannerCursor {
cur := &scannerCursor{scanner: s}
cur.scannerCursorBase = newScannerCursorBase(cur.scan, fields, opt.Location)
return cur
}
func (s *scannerCursor) scan(m map[string]interface{}) (int64, string, Tags) {
ts, name, tags := s.scanner.Peek()
// if a new series, clear the map of previous values
if name != s.series.Name || tags.ID() != s.series.Tags.ID() {
s.clear(m)
}
if ts == ZeroTime {
return ts, name, tags
}
s.scanner.ScanAt(ts, name, tags, m)
return ts, name, tags
}
func (cur *scannerCursor) Stats() IteratorStats {
return cur.scanner.Stats()
}
func (cur *scannerCursor) Err() error {
return cur.scanner.Err()
}
func (cur *scannerCursor) Close() error {
return cur.scanner.Close()
}
var _ Cursor = (*multiScannerCursor)(nil)
type multiScannerCursor struct {
scanners []IteratorScanner
err error
ascending bool
scannerCursorBase
}
func newMultiScannerCursor(scanners []IteratorScanner, fields []*influxql.Field, opt IteratorOptions) *multiScannerCursor {
cur := &multiScannerCursor{
scanners: scanners,
ascending: opt.Ascending,
}
cur.scannerCursorBase = newScannerCursorBase(cur.scan, fields, opt.Location)
return cur
}
func (cur *multiScannerCursor) scan(m map[string]interface{}) (ts int64, name string, tags Tags) {
ts = ZeroTime
for _, s := range cur.scanners {
curTime, curName, curTags := s.Peek()
if curTime == ZeroTime {
if err := s.Err(); err != nil {
cur.err = err
return ZeroTime, "", Tags{}
}
continue
}
if ts == ZeroTime {
ts, name, tags = curTime, curName, curTags
continue
}
if cur.ascending {
if (curName < name) || (curName == name && curTags.ID() < tags.ID()) || (curName == name && curTags.ID() == tags.ID() && curTime < ts) {
ts, name, tags = curTime, curName, curTags
}
continue
}
if (curName > name) || (curName == name && curTags.ID() > tags.ID()) || (curName == name && curTags.ID() == tags.ID() && curTime > ts) {
ts, name, tags = curTime, curName, curTags
}
}
if ts == ZeroTime {
return ts, name, tags
}
// if a new series, clear the map of previous values
if name != cur.series.Name || tags.ID() != cur.series.Tags.ID() {
cur.clear(m)
}
for _, s := range cur.scanners {
s.ScanAt(ts, name, tags, m)
}
return ts, name, tags
}
func (cur *multiScannerCursor) Stats() IteratorStats {
var stats IteratorStats
for _, s := range cur.scanners {
stats.Add(s.Stats())
}
return stats
}
func (cur *multiScannerCursor) Err() error {
return cur.err
}
func (cur *multiScannerCursor) Close() error {
var err error
for _, s := range cur.scanners {
if e := s.Close(); e != nil && err == nil {
err = e
}
}
return err
}
type filterCursor struct {
Cursor
// fields holds the mapping of field names to the index in the row
// based off of the column metadata. This only contains the fields
// we need and will exclude the ones we do not.
fields map[string]IteratorMap
filter influxql.Expr
m map[string]interface{}
valuer influxql.ValuerEval
}
func newFilterCursor(cur Cursor, filter influxql.Expr) *filterCursor {
fields := make(map[string]IteratorMap)
for _, name := range influxql.ExprNames(filter) {
for i, col := range cur.Columns() {
if name.Val == col.Val {
fields[name.Val] = FieldMap{
Index: i,
Type: name.Type,
}
break
}
}
// If the field is not a column, assume it is a tag value.
// We do not know what the tag values will be, but there really
// isn't any different between NullMap and a TagMap that's pointed
// at the wrong location for the purposes described here.
if _, ok := fields[name.Val]; !ok {
fields[name.Val] = TagMap(name.Val)
}
}
m := make(map[string]interface{})
return &filterCursor{
Cursor: cur,
fields: fields,
filter: filter,
m: m,
valuer: influxql.ValuerEval{Valuer: influxql.MapValuer(m)},
}
}
func (cur *filterCursor) Scan(row *Row) bool {
for cur.Cursor.Scan(row) {
// Use the field mappings to prepare the map for the valuer.
for name, f := range cur.fields {
cur.m[name] = f.Value(row)
}
if cur.valuer.EvalBool(cur.filter) {
// Passes the filter! Return true. We no longer need to
// search for a suitable value.
return true
}
}
return false
}
type nullCursor struct {
columns []influxql.VarRef
}
func newNullCursor(fields []*influxql.Field) *nullCursor {
columns := make([]influxql.VarRef, len(fields))
for i, f := range fields {
columns[i].Val = f.Name()
}
return &nullCursor{columns: columns}
}
func (cur *nullCursor) Scan(row *Row) bool {
return false
}
func (cur *nullCursor) Stats() IteratorStats {
return IteratorStats{}
}
func (cur *nullCursor) Err() error {
return nil
}
func (cur *nullCursor) Columns() []influxql.VarRef {
return cur.columns
}
func (cur *nullCursor) Close() error {
return nil
}
// DrainCursor will read and discard all values from a Cursor and return the error
// if one happens.
func DrainCursor(cur Cursor) error {
var row Row
for cur.Scan(&row) {
// Do nothing with the result.
}
return cur.Err()
}

81
influxql/query/emitter.go Normal file
View File

@ -0,0 +1,81 @@
package query
import (
"github.com/influxdata/influxdb/v2/v1/models"
)
// Emitter reads from a cursor into rows.
type Emitter struct {
cur Cursor
chunkSize int
series Series
row *models.Row
columns []string
}
// NewEmitter returns a new instance of Emitter that pulls from itrs.
func NewEmitter(cur Cursor, chunkSize int) *Emitter {
columns := make([]string, len(cur.Columns()))
for i, col := range cur.Columns() {
columns[i] = col.Val
}
return &Emitter{
cur: cur,
chunkSize: chunkSize,
columns: columns,
}
}
// Close closes the underlying iterators.
func (e *Emitter) Close() error {
return e.cur.Close()
}
// Emit returns the next row from the iterators.
func (e *Emitter) Emit() (*models.Row, bool, error) {
// Continually read from the cursor until it is exhausted.
for {
// Scan the next row. If there are no rows left, return the current row.
var row Row
if !e.cur.Scan(&row) {
if err := e.cur.Err(); err != nil {
return nil, false, err
}
r := e.row
e.row = nil
return r, false, nil
}
// If there's no row yet then create one.
// If the name and tags match the existing row, append to that row if
// the number of values doesn't exceed the chunk size.
// Otherwise return existing row and add values to next emitted row.
if e.row == nil {
e.createRow(row.Series, row.Values)
} else if e.series.SameSeries(row.Series) {
if e.chunkSize > 0 && len(e.row.Values) >= e.chunkSize {
r := e.row
r.Partial = true
e.createRow(row.Series, row.Values)
return r, true, nil
}
e.row.Values = append(e.row.Values, row.Values)
} else {
r := e.row
e.createRow(row.Series, row.Values)
return r, true, nil
}
}
}
// createRow creates a new row attached to the emitter.
func (e *Emitter) createRow(series Series, values []interface{}) {
e.series = series
e.row = &models.Row{
Name: series.Name,
Tags: series.Tags.KeyValues(),
Columns: e.columns,
Values: [][]interface{}{values},
}
}

113
influxql/query/execution_context.go Normal file
View File

@ -0,0 +1,113 @@
package query
import (
"context"
"sync"
)
// ExecutionContext contains state that the query is currently executing with.
type ExecutionContext struct {
context.Context
// The statement ID of the executing query.
statementID int
// The query ID of the executing query.
QueryID uint64
// The query task information available to the StatementExecutor.
task *Task
// Output channel where results and errors should be sent.
Results chan *Result
// Options used to start this query.
ExecutionOptions
mu sync.RWMutex
done chan struct{}
err error
}
func (ctx *ExecutionContext) watch() {
ctx.done = make(chan struct{})
if ctx.err != nil {
close(ctx.done)
return
}
go func() {
defer close(ctx.done)
var taskCtx <-chan struct{}
if ctx.task != nil {
taskCtx = ctx.task.closing
}
select {
case <-taskCtx:
ctx.err = ctx.task.Error()
if ctx.err == nil {
ctx.err = ErrQueryInterrupted
}
case <-ctx.AbortCh:
ctx.err = ErrQueryAborted
case <-ctx.Context.Done():
ctx.err = ctx.Context.Err()
}
}()
}
func (ctx *ExecutionContext) Done() <-chan struct{} {
ctx.mu.RLock()
if ctx.done != nil {
defer ctx.mu.RUnlock()
return ctx.done
}
ctx.mu.RUnlock()
ctx.mu.Lock()
defer ctx.mu.Unlock()
if ctx.done == nil {
ctx.watch()
}
return ctx.done
}
func (ctx *ExecutionContext) Err() error {
ctx.mu.RLock()
defer ctx.mu.RUnlock()
return ctx.err
}
func (ctx *ExecutionContext) Value(key interface{}) interface{} {
switch key {
case monitorContextKey{}:
return ctx.task
}
return ctx.Context.Value(key)
}
// send sends a Result to the Results channel and will exit if the query has
// been aborted.
func (ctx *ExecutionContext) send(result *Result) error {
result.StatementID = ctx.statementID
select {
case <-ctx.AbortCh:
return ErrQueryAborted
case ctx.Results <- result:
}
return nil
}
// Send sends a Result to the Results channel and will exit if the query has
// been interrupted or aborted.
func (ctx *ExecutionContext) Send(result *Result) error {
result.StatementID = ctx.statementID
select {
case <-ctx.Done():
return ctx.Err()
case ctx.Results <- result:
}
return nil
}

475
influxql/query/executor.go Normal file
View File

@ -0,0 +1,475 @@
package query
import (
"context"
"errors"
"fmt"
"os"
"runtime/debug"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/influxdata/influxdb/v2/v1/models"
"github.com/influxdata/influxql"
"go.uber.org/zap"
)
var (
// ErrInvalidQuery is returned when executing an unknown query type.
ErrInvalidQuery = errors.New("invalid query")
// ErrNotExecuted is returned when a statement is not executed in a query.
// This can occur when a previous statement in the same query has errored.
ErrNotExecuted = errors.New("not executed")
// ErrQueryInterrupted is an error returned when the query is interrupted.
ErrQueryInterrupted = errors.New("query interrupted")
// ErrQueryAborted is an error returned when the query is aborted.
ErrQueryAborted = errors.New("query aborted")
// ErrQueryEngineShutdown is an error sent when the query cannot be
// created because the query engine was shutdown.
ErrQueryEngineShutdown = errors.New("query engine shutdown")
// ErrQueryTimeoutLimitExceeded is an error when a query hits the max time allowed to run.
ErrQueryTimeoutLimitExceeded = errors.New("query-timeout limit exceeded")
// ErrAlreadyKilled is returned when attempting to kill a query that has already been killed.
ErrAlreadyKilled = errors.New("already killed")
)
// Statistics for the Executor
const (
statQueriesActive = "queriesActive" // Number of queries currently being executed.
statQueriesExecuted = "queriesExecuted" // Number of queries that have been executed (started).
statQueriesFinished = "queriesFinished" // Number of queries that have finished.
statQueryExecutionDuration = "queryDurationNs" // Total (wall) time spent executing queries.
statRecoveredPanics = "recoveredPanics" // Number of panics recovered by Query Executor.
// PanicCrashEnv is the environment variable that, when set, will prevent
// the handler from recovering any panics.
PanicCrashEnv = "INFLUXDB_PANIC_CRASH"
)
// ErrDatabaseNotFound returns a database not found error for the given database name.
func ErrDatabaseNotFound(name string) error { return fmt.Errorf("database not found: %s", name) }
// ErrMaxSelectPointsLimitExceeded is an error when a query hits the maximum number of points.
func ErrMaxSelectPointsLimitExceeded(n, limit int) error {
return fmt.Errorf("max-select-point limit exceeed: (%d/%d)", n, limit)
}
// ErrMaxConcurrentQueriesLimitExceeded is an error when a query cannot be run
// because the maximum number of queries has been reached.
func ErrMaxConcurrentQueriesLimitExceeded(n, limit int) error {
return fmt.Errorf("max-concurrent-queries limit exceeded(%d, %d)", n, limit)
}
// Authorizer determines if certain operations are authorized.
type Authorizer interface {
// AuthorizeDatabase indicates whether the given Privilege is authorized on the database with the given name.
AuthorizeDatabase(p influxql.Privilege, name string) bool
// AuthorizeQuery returns an error if the query cannot be executed
AuthorizeQuery(database string, query *influxql.Query) error
// AuthorizeSeriesRead determines if a series is authorized for reading
AuthorizeSeriesRead(database string, measurement []byte, tags models.Tags) bool
// AuthorizeSeriesWrite determines if a series is authorized for writing
AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool
}
// OpenAuthorizer is the Authorizer used when authorization is disabled.
// It allows all operations.
type openAuthorizer struct{}
// OpenAuthorizer can be shared by all goroutines.
var OpenAuthorizer = openAuthorizer{}
// AuthorizeDatabase returns true to allow any operation on a database.
func (a openAuthorizer) AuthorizeDatabase(influxql.Privilege, string) bool { return true }
// AuthorizeSeriesRead allows access to any series.
func (a openAuthorizer) AuthorizeSeriesRead(database string, measurement []byte, tags models.Tags) bool {
return true
}
// AuthorizeSeriesWrite allows access to any series.
func (a openAuthorizer) AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool {
return true
}
// AuthorizeSeriesRead allows any query to execute.
func (a openAuthorizer) AuthorizeQuery(_ string, _ *influxql.Query) error { return nil }
// AuthorizerIsOpen returns true if the provided Authorizer is guaranteed to
// authorize anything. A nil Authorizer returns true for this function, and this
// function should be preferred over directly checking if an Authorizer is nil
// or not.
func AuthorizerIsOpen(a Authorizer) bool {
if u, ok := a.(interface{ AuthorizeUnrestricted() bool }); ok {
return u.AuthorizeUnrestricted()
}
return a == nil || a == OpenAuthorizer
}
// ExecutionOptions contains the options for executing a query.
type ExecutionOptions struct {
// The database the query is running against.
Database string
// The retention policy the query is running against.
RetentionPolicy string
// How to determine whether the query is allowed to execute,
// what resources can be returned in SHOW queries, etc.
Authorizer Authorizer
// The requested maximum number of points to return in each result.
ChunkSize int
// If this query is being executed in a read-only context.
ReadOnly bool
// Node to execute on.
NodeID uint64
// Quiet suppresses non-essential output from the query executor.
Quiet bool
// AbortCh is a channel that signals when results are no longer desired by the caller.
AbortCh <-chan struct{}
}
type (
iteratorsContextKey struct{}
monitorContextKey struct{}
)
// NewContextWithIterators returns a new context.Context with the *Iterators slice added.
// The query planner will add instances of AuxIterator to the Iterators slice.
func NewContextWithIterators(ctx context.Context, itr *Iterators) context.Context {
return context.WithValue(ctx, iteratorsContextKey{}, itr)
}
// StatementExecutor executes a statement within the Executor.
type StatementExecutor interface {
// ExecuteStatement executes a statement. Results should be sent to the
// results channel in the ExecutionContext.
ExecuteStatement(stmt influxql.Statement, ctx *ExecutionContext) error
}
// StatementNormalizer normalizes a statement before it is executed.
type StatementNormalizer interface {
// NormalizeStatement adds a default database and policy to the
// measurements in the statement.
NormalizeStatement(stmt influxql.Statement, database, retentionPolicy string) error
}
// Executor executes every statement in an Query.
type Executor struct {
// Used for executing a statement in the query.
StatementExecutor StatementExecutor
// Used for tracking running queries.
TaskManager *TaskManager
// Logger to use for all logging.
// Defaults to discarding all log output.
Logger *zap.Logger
// expvar-based stats.
stats *Statistics
}
// NewExecutor returns a new instance of Executor.
func NewExecutor() *Executor {
return &Executor{
TaskManager: NewTaskManager(),
Logger: zap.NewNop(),
stats: &Statistics{},
}
}
// Statistics keeps statistics related to the Executor.
type Statistics struct {
ActiveQueries int64
ExecutedQueries int64
FinishedQueries int64
QueryExecutionDuration int64
RecoveredPanics int64
}
// Statistics returns statistics for periodic monitoring.
func (e *Executor) Statistics(tags map[string]string) []models.Statistic {
return []models.Statistic{{
Name: "queryExecutor",
Tags: tags,
Values: map[string]interface{}{
statQueriesActive: atomic.LoadInt64(&e.stats.ActiveQueries),
statQueriesExecuted: atomic.LoadInt64(&e.stats.ExecutedQueries),
statQueriesFinished: atomic.LoadInt64(&e.stats.FinishedQueries),
statQueryExecutionDuration: atomic.LoadInt64(&e.stats.QueryExecutionDuration),
statRecoveredPanics: atomic.LoadInt64(&e.stats.RecoveredPanics),
},
}}
}
// Close kills all running queries and prevents new queries from being attached.
func (e *Executor) Close() error {
return e.TaskManager.Close()
}
// SetLogOutput sets the writer to which all logs are written. It must not be
// called after Open is called.
func (e *Executor) WithLogger(log *zap.Logger) {
e.Logger = log.With(zap.String("service", "query"))
e.TaskManager.Logger = e.Logger
}
// ExecuteQuery executes each statement within a query.
func (e *Executor) ExecuteQuery(query *influxql.Query, opt ExecutionOptions, closing chan struct{}) <-chan *Result {
results := make(chan *Result)
go e.executeQuery(query, opt, closing, results)
return results
}
func (e *Executor) executeQuery(query *influxql.Query, opt ExecutionOptions, closing <-chan struct{}, results chan *Result) {
defer close(results)
defer e.recover(query, results)
atomic.AddInt64(&e.stats.ActiveQueries, 1)
atomic.AddInt64(&e.stats.ExecutedQueries, 1)
defer func(start time.Time) {
atomic.AddInt64(&e.stats.ActiveQueries, -1)
atomic.AddInt64(&e.stats.FinishedQueries, 1)
atomic.AddInt64(&e.stats.QueryExecutionDuration, time.Since(start).Nanoseconds())
}(time.Now())
ctx, detach, err := e.TaskManager.AttachQuery(query, opt, closing)
if err != nil {
select {
case results <- &Result{Err: err}:
case <-opt.AbortCh:
}
return
}
defer detach()
// Setup the execution context that will be used when executing statements.
ctx.Results = results
var i int
LOOP:
for ; i < len(query.Statements); i++ {
ctx.statementID = i
stmt := query.Statements[i]
// If a default database wasn't passed in by the caller, check the statement.
defaultDB := opt.Database
if defaultDB == "" {
if s, ok := stmt.(influxql.HasDefaultDatabase); ok {
defaultDB = s.DefaultDatabase()
}
}
// Do not let queries manually use the system measurements. If we find
// one, return an error. This prevents a person from using the
// measurement incorrectly and causing a panic.
if stmt, ok := stmt.(*influxql.SelectStatement); ok {
for _, s := range stmt.Sources {
switch s := s.(type) {
case *influxql.Measurement:
if influxql.IsSystemName(s.Name) {
command := "the appropriate meta command"
switch s.Name {
case "_fieldKeys":
command = "SHOW FIELD KEYS"
case "_measurements":
command = "SHOW MEASUREMENTS"
case "_series":
command = "SHOW SERIES"
case "_tagKeys":
command = "SHOW TAG KEYS"
case "_tags":
command = "SHOW TAG VALUES"
}
results <- &Result{
Err: fmt.Errorf("unable to use system source '%s': use %s instead", s.Name, command),
}
break LOOP
}
}
}
}
// Rewrite statements, if necessary.
// This can occur on meta read statements which convert to SELECT statements.
newStmt, err := RewriteStatement(stmt)
if err != nil {
results <- &Result{Err: err}
break
}
stmt = newStmt
// Normalize each statement if possible.
if normalizer, ok := e.StatementExecutor.(StatementNormalizer); ok {
if err := normalizer.NormalizeStatement(stmt, defaultDB, opt.RetentionPolicy); err != nil {
if err := ctx.send(&Result{Err: err}); err == ErrQueryAborted {
return
}
break
}
}
// Log each normalized statement.
if !ctx.Quiet {
e.Logger.Info("Executing query", zap.Stringer("query", stmt))
}
// Send any other statements to the underlying statement executor.
err = e.StatementExecutor.ExecuteStatement(stmt, ctx)
if err == ErrQueryInterrupted {
// Query was interrupted so retrieve the real interrupt error from
// the query task if there is one.
if qerr := ctx.Err(); qerr != nil {
err = qerr
}
}
// Send an error for this result if it failed for some reason.
if err != nil {
if err := ctx.send(&Result{
StatementID: i,
Err: err,
}); err == ErrQueryAborted {
return
}
// Stop after the first error.
break
}
// Check if the query was interrupted during an uninterruptible statement.
interrupted := false
select {
case <-ctx.Done():
interrupted = true
default:
// Query has not been interrupted.
}
if interrupted {
break
}
}
// Send error results for any statements which were not executed.
for ; i < len(query.Statements)-1; i++ {
if err := ctx.send(&Result{
StatementID: i,
Err: ErrNotExecuted,
}); err == ErrQueryAborted {
return
}
}
}
// Determines if the Executor will recover any panics or let them crash
// the server.
var willCrash bool
func init() {
var err error
if willCrash, err = strconv.ParseBool(os.Getenv(PanicCrashEnv)); err != nil {
willCrash = false
}
}
func (e *Executor) recover(query *influxql.Query, results chan *Result) {
if err := recover(); err != nil {
atomic.AddInt64(&e.stats.RecoveredPanics, 1) // Capture the panic in _internal stats.
e.Logger.Error(fmt.Sprintf("%s [panic:%s] %s", query.String(), err, debug.Stack()))
results <- &Result{
StatementID: -1,
Err: fmt.Errorf("%s [panic:%s]", query.String(), err),
}
if willCrash {
e.Logger.Error(fmt.Sprintf("\n\n=====\nAll goroutines now follow:"))
buf := debug.Stack()
e.Logger.Error(fmt.Sprintf("%s", buf))
os.Exit(1)
}
}
}
// Task is the internal data structure for managing queries.
// For the public use data structure that gets returned, see Task.
type Task struct {
query string
database string
status TaskStatus
startTime time.Time
closing chan struct{}
monitorCh chan error
err error
mu sync.Mutex
}
// Monitor starts a new goroutine that will monitor a query. The function
// will be passed in a channel to signal when the query has been finished
// normally. If the function returns with an error and the query is still
// running, the query will be terminated.
func (q *Task) Monitor(fn MonitorFunc) {
go q.monitor(fn)
}
// Error returns any asynchronous error that may have occurred while executing
// the query.
func (q *Task) Error() error {
q.mu.Lock()
defer q.mu.Unlock()
return q.err
}
func (q *Task) setError(err error) {
q.mu.Lock()
q.err = err
q.mu.Unlock()
}
func (q *Task) monitor(fn MonitorFunc) {
if err := fn(q.closing); err != nil {
select {
case <-q.closing:
case q.monitorCh <- err:
}
}
}
// close closes the query task closing channel if the query hasn't been previously killed.
func (q *Task) close() {
q.mu.Lock()
if q.status != KilledTask {
// Set the status to killed to prevent closing the channel twice.
q.status = KilledTask
close(q.closing)
}
q.mu.Unlock()
}
func (q *Task) kill() error {
q.mu.Lock()
if q.status == KilledTask {
q.mu.Unlock()
return ErrAlreadyKilled
}
q.status = KilledTask
close(q.closing)
q.mu.Unlock()
return nil
}

535
influxql/query/executor_test.go Normal file
View File

@ -0,0 +1,535 @@
package query_test
import (
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxql"
)
var errUnexpected = errors.New("unexpected error")
type StatementExecutor struct {
ExecuteStatementFn func(stmt influxql.Statement, ctx *query.ExecutionContext) error
}
func (e *StatementExecutor) ExecuteStatement(stmt influxql.Statement, ctx *query.ExecutionContext) error {
return e.ExecuteStatementFn(stmt, ctx)
}
func NewQueryExecutor() *query.Executor {
return query.NewExecutor()
}
func TestQueryExecutor_AttachQuery(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
if ctx.QueryID != 1 {
t.Errorf("incorrect query id: exp=1 got=%d", ctx.QueryID)
}
return nil
},
}
discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
}
func TestQueryExecutor_KillQuery(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
qid := make(chan uint64)
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
switch stmt.(type) {
case *influxql.KillQueryStatement:
return e.TaskManager.ExecuteStatement(stmt, ctx)
}
qid <- ctx.QueryID
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(100 * time.Millisecond):
t.Error("killing the query did not close the channel after 100 milliseconds")
return errUnexpected
}
},
}
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
q, err = influxql.ParseQuery(fmt.Sprintf("KILL QUERY %d", <-qid))
if err != nil {
t.Fatal(err)
}
discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
result := <-results
if result.Err != query.ErrQueryInterrupted {
t.Errorf("unexpected error: %s", result.Err)
}
}
func TestQueryExecutor_KillQuery_Zombie(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
qid := make(chan uint64)
done := make(chan struct{})
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
switch stmt.(type) {
case *influxql.KillQueryStatement, *influxql.ShowQueriesStatement:
return e.TaskManager.ExecuteStatement(stmt, ctx)
}
qid <- ctx.QueryID
select {
case <-ctx.Done():
select {
case <-done:
// Keep the query running until we run SHOW QUERIES.
case <-time.After(100 * time.Millisecond):
// Ensure that we don't have a lingering goroutine.
}
return query.ErrQueryInterrupted
case <-time.After(100 * time.Millisecond):
t.Error("killing the query did not close the channel after 100 milliseconds")
return errUnexpected
}
},
}
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
q, err = influxql.ParseQuery(fmt.Sprintf("KILL QUERY %d", <-qid))
if err != nil {
t.Fatal(err)
}
discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
// Display the queries and ensure that the original is still in there.
q, err = influxql.ParseQuery("SHOW QUERIES")
if err != nil {
t.Fatal(err)
}
tasks := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
// The killed query should still be there.
task := <-tasks
if len(task.Series) != 1 {
t.Errorf("expected %d series, got %d", 1, len(task.Series))
} else if len(task.Series[0].Values) != 2 {
t.Errorf("expected %d rows, got %d", 2, len(task.Series[0].Values))
}
close(done)
// The original query should return.
result := <-results
if result.Err != query.ErrQueryInterrupted {
t.Errorf("unexpected error: %s", result.Err)
}
}
func TestQueryExecutor_KillQuery_CloseTaskManager(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
qid := make(chan uint64)
// Open a channel to stall the statement executor forever. This keeps the statement executor
// running even after we kill the query which can happen with some queries. We only close it once
// the test has finished running.
done := make(chan struct{})
defer close(done)
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
switch stmt.(type) {
case *influxql.KillQueryStatement, *influxql.ShowQueriesStatement:
return e.TaskManager.ExecuteStatement(stmt, ctx)
}
qid <- ctx.QueryID
<-done
return nil
},
}
// Kill the query. This should switch it into a zombie state.
go discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
q, err = influxql.ParseQuery(fmt.Sprintf("KILL QUERY %d", <-qid))
if err != nil {
t.Fatal(err)
}
discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
// Display the queries and ensure that the original is still in there.
q, err = influxql.ParseQuery("SHOW QUERIES")
if err != nil {
t.Fatal(err)
}
tasks := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
// The killed query should still be there.
task := <-tasks
if len(task.Series) != 1 {
t.Errorf("expected %d series, got %d", 1, len(task.Series))
} else if len(task.Series[0].Values) != 2 {
t.Errorf("expected %d rows, got %d", 2, len(task.Series[0].Values))
}
// Close the task manager to ensure it doesn't cause a panic.
if err := e.TaskManager.Close(); err != nil {
t.Errorf("unexpected error: %s", err)
}
}
func TestQueryExecutor_KillQuery_AlreadyKilled(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
qid := make(chan uint64)
// Open a channel to stall the statement executor forever. This keeps the statement executor
// running even after we kill the query which can happen with some queries. We only close it once
// the test has finished running.
done := make(chan struct{})
defer close(done)
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
switch stmt.(type) {
case *influxql.KillQueryStatement, *influxql.ShowQueriesStatement:
return e.TaskManager.ExecuteStatement(stmt, ctx)
}
qid <- ctx.QueryID
<-done
return nil
},
}
// Kill the query. This should switch it into a zombie state.
go discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
q, err = influxql.ParseQuery(fmt.Sprintf("KILL QUERY %d", <-qid))
if err != nil {
t.Fatal(err)
}
discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
// Now attempt to kill it again. We should get an error.
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
result := <-results
if got, want := result.Err, query.ErrAlreadyKilled; got != want {
t.Errorf("unexpected error: got=%v want=%v", got, want)
}
}
func TestQueryExecutor_Interrupt(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(100 * time.Millisecond):
t.Error("killing the query did not close the channel after 100 milliseconds")
return errUnexpected
}
},
}
closing := make(chan struct{})
results := e.ExecuteQuery(q, query.ExecutionOptions{}, closing)
close(closing)
result := <-results
if result.Err != query.ErrQueryInterrupted {
t.Errorf("unexpected error: %s", result.Err)
}
}
func TestQueryExecutor_Abort(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
ch1 := make(chan struct{})
ch2 := make(chan struct{})
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
<-ch1
if err := ctx.Send(&query.Result{Err: errUnexpected}); err != query.ErrQueryAborted {
t.Errorf("unexpected error: %v", err)
}
close(ch2)
return nil
},
}
done := make(chan struct{})
close(done)
results := e.ExecuteQuery(q, query.ExecutionOptions{AbortCh: done}, nil)
close(ch1)
<-ch2
discardOutput(results)
}
func TestQueryExecutor_ShowQueries(t *testing.T) {
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
switch stmt.(type) {
case *influxql.ShowQueriesStatement:
return e.TaskManager.ExecuteStatement(stmt, ctx)
}
t.Errorf("unexpected statement: %s", stmt)
return errUnexpected
},
}
q, err := influxql.ParseQuery(`SHOW QUERIES`)
if err != nil {
t.Fatal(err)
}
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
result := <-results
if len(result.Series) != 1 {
t.Errorf("expected %d series, got %d", 1, len(result.Series))
} else if len(result.Series[0].Values) != 1 {
t.Errorf("expected %d row, got %d", 1, len(result.Series[0].Values))
}
if result.Err != nil {
t.Errorf("unexpected error: %s", result.Err)
}
}
func TestQueryExecutor_Limit_Timeout(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Second):
t.Errorf("timeout has not killed the query")
return errUnexpected
}
},
}
e.TaskManager.QueryTimeout = time.Nanosecond
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
result := <-results
if result.Err == nil || !strings.Contains(result.Err.Error(), "query-timeout") {
t.Errorf("unexpected error: %s", result.Err)
}
}
func TestQueryExecutor_Limit_ConcurrentQueries(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
qid := make(chan uint64)
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
qid <- ctx.QueryID
<-ctx.Done()
return ctx.Err()
},
}
e.TaskManager.MaxConcurrentQueries = 1
defer e.Close()
// Start first query and wait for it to be executing.
go discardOutput(e.ExecuteQuery(q, query.ExecutionOptions{}, nil))
<-qid
// Start second query and expect for it to fail.
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
select {
case result := <-results:
if len(result.Series) != 0 {
t.Errorf("expected %d rows, got %d", 0, len(result.Series))
}
if result.Err == nil || !strings.Contains(result.Err.Error(), "max-concurrent-queries") {
t.Errorf("unexpected error: %s", result.Err)
}
case <-qid:
t.Errorf("unexpected statement execution for the second query")
}
}
func TestQueryExecutor_Close(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
ch1 := make(chan struct{})
ch2 := make(chan struct{})
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
close(ch1)
<-ctx.Done()
return ctx.Err()
},
}
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
go func(results <-chan *query.Result) {
result := <-results
if result.Err != query.ErrQueryEngineShutdown {
t.Errorf("unexpected error: %s", result.Err)
}
close(ch2)
}(results)
// Wait for the statement to start executing.
<-ch1
// Close the query executor.
e.Close()
// Check that the statement gets interrupted and finishes.
select {
case <-ch2:
case <-time.After(100 * time.Millisecond):
t.Fatal("closing the query manager did not kill the query after 100 milliseconds")
}
results = e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
result := <-results
if len(result.Series) != 0 {
t.Errorf("expected %d rows, got %d", 0, len(result.Series))
}
if result.Err != query.ErrQueryEngineShutdown {
t.Errorf("unexpected error: %s", result.Err)
}
}
func TestQueryExecutor_Panic(t *testing.T) {
q, err := influxql.ParseQuery(`SELECT count(value) FROM cpu`)
if err != nil {
t.Fatal(err)
}
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
panic("test error")
},
}
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
result := <-results
if len(result.Series) != 0 {
t.Errorf("expected %d rows, got %d", 0, len(result.Series))
}
if result.Err == nil || result.Err.Error() != "SELECT count(value) FROM cpu [panic:test error]" {
t.Errorf("unexpected error: %s", result.Err)
}
}
func TestQueryExecutor_InvalidSource(t *testing.T) {
e := NewQueryExecutor()
e.StatementExecutor = &StatementExecutor{
ExecuteStatementFn: func(stmt influxql.Statement, ctx *query.ExecutionContext) error {
return errors.New("statement executed unexpectedly")
},
}
for i, tt := range []struct {
q string
err string
}{
{
q: `SELECT fieldKey, fieldType FROM _fieldKeys`,
err: `unable to use system source '_fieldKeys': use SHOW FIELD KEYS instead`,
},
{
q: `SELECT "name" FROM _measurements`,
err: `unable to use system source '_measurements': use SHOW MEASUREMENTS instead`,
},
{
q: `SELECT "key" FROM _series`,
err: `unable to use system source '_series': use SHOW SERIES instead`,
},
{
q: `SELECT tagKey FROM _tagKeys`,
err: `unable to use system source '_tagKeys': use SHOW TAG KEYS instead`,
},
{
q: `SELECT "key", value FROM _tags`,
err: `unable to use system source '_tags': use SHOW TAG VALUES instead`,
},
} {
q, err := influxql.ParseQuery(tt.q)
if err != nil {
t.Errorf("%d. unable to parse: %s", i, tt.q)
continue
}
results := e.ExecuteQuery(q, query.ExecutionOptions{}, nil)
result := <-results
if len(result.Series) != 0 {
t.Errorf("%d. expected %d rows, got %d", 0, i, len(result.Series))
}
if result.Err == nil || result.Err.Error() != tt.err {
t.Errorf("%d. unexpected error: %s", i, result.Err)
}
}
}
func discardOutput(results <-chan *query.Result) {
for range results {
// Read all results and discard.
}
}

86
influxql/query/explain.go Normal file
View File

@ -0,0 +1,86 @@
package query
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"github.com/influxdata/influxql"
)
func (p *preparedStatement) Explain() (string, error) {
// Determine the cost of all iterators created as part of this plan.
ic := &explainIteratorCreator{ic: p.ic}
p.ic = ic
cur, err := p.Select(context.Background())
p.ic = ic.ic
if err != nil {
return "", err
}
cur.Close()
var buf bytes.Buffer
for i, node := range ic.nodes {
if i > 0 {
buf.WriteString("\n")
}
expr := "<nil>"
if node.Expr != nil {
expr = node.Expr.String()
}
fmt.Fprintf(&buf, "EXPRESSION: %s\n", expr)
if len(node.Aux) != 0 {
refs := make([]string, len(node.Aux))
for i, ref := range node.Aux {
refs[i] = ref.String()
}
fmt.Fprintf(&buf, "AUXILIARY FIELDS: %s\n", strings.Join(refs, ", "))
}
fmt.Fprintf(&buf, "NUMBER OF SHARDS: %d\n", node.Cost.NumShards)
fmt.Fprintf(&buf, "NUMBER OF SERIES: %d\n", node.Cost.NumSeries)
fmt.Fprintf(&buf, "CACHED VALUES: %d\n", node.Cost.CachedValues)
fmt.Fprintf(&buf, "NUMBER OF FILES: %d\n", node.Cost.NumFiles)
fmt.Fprintf(&buf, "NUMBER OF BLOCKS: %d\n", node.Cost.BlocksRead)
fmt.Fprintf(&buf, "SIZE OF BLOCKS: %d\n", node.Cost.BlockSize)
}
return buf.String(), nil
}
type planNode struct {
Expr influxql.Expr
Aux []influxql.VarRef
Cost IteratorCost
}
type explainIteratorCreator struct {
ic interface {
IteratorCreator
io.Closer
}
nodes []planNode
}
func (e *explainIteratorCreator) CreateIterator(ctx context.Context, m *influxql.Measurement, opt IteratorOptions) (Iterator, error) {
cost, err := e.ic.IteratorCost(m, opt)
if err != nil {
return nil, err
}
e.nodes = append(e.nodes, planNode{
Expr: opt.Expr,
Aux: opt.Aux,
Cost: cost,
})
return &nilFloatIterator{}, nil
}
func (e *explainIteratorCreator) IteratorCost(m *influxql.Measurement, opt IteratorOptions) (IteratorCost, error) {
return e.ic.IteratorCost(m, opt)
}
func (e *explainIteratorCreator) Close() error {
return e.ic.Close()
}

2433
influxql/query/functions.gen.go Normal file
View File

File diff suppressed because it is too large Load Diff

219
influxql/query/functions.gen.go.tmpl Normal file
View File

@ -0,0 +1,219 @@
package query
import (
"sort"
"time"
"math/rand"
)
{{with $types := .}}{{range $k := $types}}
// {{$k.Name}}PointAggregator aggregates points to produce a single point.
type {{$k.Name}}PointAggregator interface {
Aggregate{{$k.Name}}(p *{{$k.Name}}Point)
}
// {{$k.Name}}BulkPointAggregator aggregates multiple points at a time.
type {{$k.Name}}BulkPointAggregator interface {
Aggregate{{$k.Name}}Bulk(points []{{$k.Name}}Point)
}
// Aggregate{{$k.Name}}Points feeds a slice of {{$k.Name}}Point into an
// aggregator. If the aggregator is a {{$k.Name}}BulkPointAggregator, it will
// use the AggregateBulk method.
func Aggregate{{$k.Name}}Points(a {{$k.Name}}PointAggregator, points []{{$k.Name}}Point) {
switch a := a.(type) {
case {{$k.Name}}BulkPointAggregator:
a.Aggregate{{$k.Name}}Bulk(points)
default:
for _, p := range points {
a.Aggregate{{$k.Name}}(&p)
}
}
}
// {{$k.Name}}PointEmitter produces a single point from an aggregate.
type {{$k.Name}}PointEmitter interface {
Emit() []{{$k.Name}}Point
}
{{range $v := $types}}
// {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Func is the function called by a {{$k.Name}}Point reducer.
type {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Func func(prev *{{$v.Name}}Point, curr *{{$k.Name}}Point) (t int64, v {{$v.Type}}, aux []interface{})
// {{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer is a reducer that reduces
// the passed in points to a single point using a reduce function.
type {{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer struct {
prev *{{$v.Name}}Point
fn {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Func
}
// New{{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer creates a new {{$k.Name}}Func{{$v.Name}}Reducer.
func New{{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer(fn {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Func, prev *{{$v.Name}}Point) *{{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer {
return &{{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer{fn: fn, prev: prev}
}
// Aggregate{{$k.Name}} takes a {{$k.Name}}Point and invokes the reduce function with the
// current and new point to modify the current point.
func (r *{{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer) Aggregate{{$k.Name}}(p *{{$k.Name}}Point) {
t, v, aux := r.fn(r.prev, p)
if r.prev == nil {
r.prev = &{{$v.Name}}Point{}
}
r.prev.Time = t
r.prev.Value = v
r.prev.Aux = aux
if p.Aggregated > 1 {
r.prev.Aggregated += p.Aggregated
} else {
r.prev.Aggregated++
}
}
// Emit emits the point that was generated when reducing the points fed in with Aggregate{{$k.Name}}.
func (r *{{$k.Name}}Func{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer) Emit() []{{$v.Name}}Point {
return []{{$v.Name}}Point{*r.prev}
}
// {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}SliceFunc is the function called by a {{$k.Name}}Point reducer.
type {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}SliceFunc func(a []{{$k.Name}}Point) []{{$v.Name}}Point
// {{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer is a reducer that aggregates
// the passed in points and then invokes the function to reduce the points when they are emitted.
type {{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer struct {
points []{{$k.Name}}Point
fn {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}SliceFunc
}
// New{{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer creates a new {{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer.
func New{{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer(fn {{$k.Name}}Reduce{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}SliceFunc) *{{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer {
return &{{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer{fn: fn}
}
// Aggregate{{$k.Name}} copies the {{$k.Name}}Point into the internal slice to be passed
// to the reduce function when Emit is called.
func (r *{{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer) Aggregate{{$k.Name}}(p *{{$k.Name}}Point) {
r.points = append(r.points, *p.Clone())
}
// Aggregate{{$k.Name}}Bulk performs a bulk copy of {{$k.Name}}Points into the internal slice.
// This is a more efficient version of calling Aggregate{{$k.Name}} on each point.
func (r *{{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer) Aggregate{{$k.Name}}Bulk(points []{{$k.Name}}Point) {
r.points = append(r.points, points...)
}
// Emit invokes the reduce function on the aggregated points to generate the aggregated points.
// This method does not clear the points from the internal slice.
func (r *{{$k.Name}}SliceFunc{{if ne $k.Name $v.Name}}{{$v.Name}}{{end}}Reducer) Emit() []{{$v.Name}}Point {
return r.fn(r.points)
}
{{end}}
// {{$k.Name}}DistinctReducer returns the distinct points in a series.
type {{$k.Name}}DistinctReducer struct {
m map[{{$k.Type}}]{{$k.Name}}Point
}
// New{{$k.Name}}DistinctReducer creates a new {{$k.Name}}DistinctReducer.
func New{{$k.Name}}DistinctReducer() *{{$k.Name}}DistinctReducer {
return &{{$k.Name}}DistinctReducer{m: make(map[{{$k.Type}}]{{$k.Name}}Point)}
}
// Aggregate{{$k.Name}} aggregates a point into the reducer.
func (r *{{$k.Name}}DistinctReducer) Aggregate{{$k.Name}}(p *{{$k.Name}}Point) {
if _, ok := r.m[p.Value]; !ok {
r.m[p.Value] = *p
}
}
// Emit emits the distinct points that have been aggregated into the reducer.
func (r *{{$k.Name}}DistinctReducer) Emit() []{{$k.Name}}Point {
points := make([]{{$k.Name}}Point, 0, len(r.m))
for _, p := range r.m {
points = append(points, {{$k.Name}}Point{Time: p.Time, Value: p.Value})
}
sort.Sort({{$k.name}}Points(points))
return points
}
// {{$k.Name}}ElapsedReducer calculates the elapsed of the aggregated points.
type {{$k.Name}}ElapsedReducer struct {
unitConversion int64
prev {{$k.Name}}Point
curr {{$k.Name}}Point
}
// New{{$k.Name}}ElapsedReducer creates a new {{$k.Name}}ElapsedReducer.
func New{{$k.Name}}ElapsedReducer(interval Interval) *{{$k.Name}}ElapsedReducer {
return &{{$k.Name}}ElapsedReducer{
unitConversion: int64(interval.Duration),
prev: {{$k.Name}}Point{Nil: true},
curr: {{$k.Name}}Point{Nil: true},
}
}
// Aggregate{{$k.Name}} aggregates a point into the reducer and updates the current window.
func (r *{{$k.Name}}ElapsedReducer) Aggregate{{$k.Name}}(p *{{$k.Name}}Point) {
r.prev = r.curr
r.curr = *p
}
// Emit emits the elapsed of the reducer at the current point.
func (r *{{$k.Name}}ElapsedReducer) Emit() []IntegerPoint {
if !r.prev.Nil {
elapsed := (r.curr.Time - r.prev.Time) / r.unitConversion
return []IntegerPoint{
{Time: r.curr.Time, Value: elapsed},
}
}
return nil
}
// {{$k.Name}}SampleReducer implements a reservoir sampling to calculate a random subset of points
type {{$k.Name}}SampleReducer struct {
count int // how many points we've iterated over
rng *rand.Rand // random number generator for each reducer
points {{$k.name}}Points // the reservoir
}
// New{{$k.Name}}SampleReducer creates a new {{$k.Name}}SampleReducer
func New{{$k.Name}}SampleReducer(size int) *{{$k.Name}}SampleReducer {
return &{{$k.Name}}SampleReducer{
rng: rand.New(rand.NewSource(time.Now().UnixNano())), // seed with current time as suggested by https://golang.org/pkg/math/rand/
points: make({{$k.name}}Points, size),
}
}
// Aggregate{{$k.Name}} aggregates a point into the reducer.
func (r *{{$k.Name}}SampleReducer) Aggregate{{$k.Name}}(p *{{$k.Name}}Point) {
r.count++
// Fill the reservoir with the first n points
if r.count-1 < len(r.points) {
p.CopyTo(&r.points[r.count-1])
return
}
// Generate a random integer between 1 and the count and
// if that number is less than the length of the slice
// replace the point at that index rnd with p.
rnd := r.rng.Intn(r.count)
if rnd < len(r.points) {
p.CopyTo(&r.points[rnd])
}
}
// Emit emits the reservoir sample as many points.
func (r *{{$k.Name}}SampleReducer) Emit() []{{$k.Name}}Point {
min := len(r.points)
if r.count < min {
min = r.count
}
pts := r.points[:min]
sort.Sort(pts)
return pts
}
{{end}}{{end}}

2152
influxql/query/functions.go Normal file
View File

File diff suppressed because it is too large Load Diff

499
influxql/query/functions_test.go Normal file
View File

@ -0,0 +1,499 @@
package query_test
import (
"math"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/pkg/deep"
"github.com/influxdata/influxql"
)
func almostEqual(got, exp float64) bool {
return math.Abs(got-exp) < 1e-5 && !math.IsNaN(got)
}
func TestHoltWinters_AusTourists(t *testing.T) {
hw := query.NewFloatHoltWintersReducer(10, 4, false, 1)
// Dataset from http://www.inside-r.org/packages/cran/fpp/docs/austourists
austourists := []query.FloatPoint{
{Time: 1, Value: 30.052513},
{Time: 2, Value: 19.148496},
{Time: 3, Value: 25.317692},
{Time: 4, Value: 27.591437},
{Time: 5, Value: 32.076456},
{Time: 6, Value: 23.487961},
{Time: 7, Value: 28.47594},
{Time: 8, Value: 35.123753},
{Time: 9, Value: 36.838485},
{Time: 10, Value: 25.007017},
{Time: 11, Value: 30.72223},
{Time: 12, Value: 28.693759},
{Time: 13, Value: 36.640986},
{Time: 14, Value: 23.824609},
{Time: 15, Value: 29.311683},
{Time: 16, Value: 31.770309},
{Time: 17, Value: 35.177877},
{Time: 18, Value: 19.775244},
{Time: 19, Value: 29.60175},
{Time: 20, Value: 34.538842},
{Time: 21, Value: 41.273599},
{Time: 22, Value: 26.655862},
{Time: 23, Value: 28.279859},
{Time: 24, Value: 35.191153},
{Time: 25, Value: 41.727458},
{Time: 26, Value: 24.04185},
{Time: 27, Value: 32.328103},
{Time: 28, Value: 37.328708},
{Time: 29, Value: 46.213153},
{Time: 30, Value: 29.346326},
{Time: 31, Value: 36.48291},
{Time: 32, Value: 42.977719},
{Time: 33, Value: 48.901525},
{Time: 34, Value: 31.180221},
{Time: 35, Value: 37.717881},
{Time: 36, Value: 40.420211},
{Time: 37, Value: 51.206863},
{Time: 38, Value: 31.887228},
{Time: 39, Value: 40.978263},
{Time: 40, Value: 43.772491},
{Time: 41, Value: 55.558567},
{Time: 42, Value: 33.850915},
{Time: 43, Value: 42.076383},
{Time: 44, Value: 45.642292},
{Time: 45, Value: 59.76678},
{Time: 46, Value: 35.191877},
{Time: 47, Value: 44.319737},
{Time: 48, Value: 47.913736},
}
for _, p := range austourists {
hw.AggregateFloat(&p)
}
points := hw.Emit()
forecasted := []query.FloatPoint{
{Time: 49, Value: 51.85064132137853},
{Time: 50, Value: 43.26055282315273},
{Time: 51, Value: 41.827258044814464},
{Time: 52, Value: 54.3990354591749},
{Time: 53, Value: 54.62334472770803},
{Time: 54, Value: 45.57155693625209},
{Time: 55, Value: 44.06051240252263},
{Time: 56, Value: 57.30029870759433},
{Time: 57, Value: 57.53591513519172},
{Time: 58, Value: 47.999008139396096},
}
if exp, got := len(forecasted), len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
for i := range forecasted {
if exp, got := forecasted[i].Time, points[i].Time; got != exp {
t.Errorf("unexpected time on points[%d] got %v exp %v", i, got, exp)
}
if exp, got := forecasted[i].Value, points[i].Value; !almostEqual(got, exp) {
t.Errorf("unexpected value on points[%d] got %v exp %v", i, got, exp)
}
}
}
func TestHoltWinters_AusTourists_Missing(t *testing.T) {
hw := query.NewFloatHoltWintersReducer(10, 4, false, 1)
// Dataset from http://www.inside-r.org/packages/cran/fpp/docs/austourists
austourists := []query.FloatPoint{
{Time: 1, Value: 30.052513},
{Time: 3, Value: 25.317692},
{Time: 4, Value: 27.591437},
{Time: 5, Value: 32.076456},
{Time: 6, Value: 23.487961},
{Time: 7, Value: 28.47594},
{Time: 9, Value: 36.838485},
{Time: 10, Value: 25.007017},
{Time: 11, Value: 30.72223},
{Time: 12, Value: 28.693759},
{Time: 13, Value: 36.640986},
{Time: 14, Value: 23.824609},
{Time: 15, Value: 29.311683},
{Time: 16, Value: 31.770309},
{Time: 17, Value: 35.177877},
{Time: 19, Value: 29.60175},
{Time: 20, Value: 34.538842},
{Time: 21, Value: 41.273599},
{Time: 22, Value: 26.655862},
{Time: 23, Value: 28.279859},
{Time: 24, Value: 35.191153},
{Time: 25, Value: 41.727458},
{Time: 26, Value: 24.04185},
{Time: 27, Value: 32.328103},
{Time: 28, Value: 37.328708},
{Time: 30, Value: 29.346326},
{Time: 31, Value: 36.48291},
{Time: 32, Value: 42.977719},
{Time: 34, Value: 31.180221},
{Time: 35, Value: 37.717881},
{Time: 36, Value: 40.420211},
{Time: 37, Value: 51.206863},
{Time: 38, Value: 31.887228},
{Time: 41, Value: 55.558567},
{Time: 42, Value: 33.850915},
{Time: 43, Value: 42.076383},
{Time: 44, Value: 45.642292},
{Time: 45, Value: 59.76678},
{Time: 46, Value: 35.191877},
{Time: 47, Value: 44.319737},
{Time: 48, Value: 47.913736},
}
for _, p := range austourists {
hw.AggregateFloat(&p)
}
points := hw.Emit()
forecasted := []query.FloatPoint{
{Time: 49, Value: 54.84533610387743},
{Time: 50, Value: 41.19329421863249},
{Time: 51, Value: 45.71673175112451},
{Time: 52, Value: 56.05759298805955},
{Time: 53, Value: 59.32337460282217},
{Time: 54, Value: 44.75280096850461},
{Time: 55, Value: 49.98865098113751},
{Time: 56, Value: 61.86084934967605},
{Time: 57, Value: 65.95805633454883},
{Time: 58, Value: 50.1502170480547},
}
if exp, got := len(forecasted), len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
for i := range forecasted {
if exp, got := forecasted[i].Time, points[i].Time; got != exp {
t.Errorf("unexpected time on points[%d] got %v exp %v", i, got, exp)
}
if exp, got := forecasted[i].Value, points[i].Value; !almostEqual(got, exp) {
t.Errorf("unexpected value on points[%d] got %v exp %v", i, got, exp)
}
}
}
func TestHoltWinters_USPopulation(t *testing.T) {
series := []query.FloatPoint{
{Time: 1, Value: 3.93},
{Time: 2, Value: 5.31},
{Time: 3, Value: 7.24},
{Time: 4, Value: 9.64},
{Time: 5, Value: 12.90},
{Time: 6, Value: 17.10},
{Time: 7, Value: 23.20},
{Time: 8, Value: 31.40},
{Time: 9, Value: 39.80},
{Time: 10, Value: 50.20},
{Time: 11, Value: 62.90},
{Time: 12, Value: 76.00},
{Time: 13, Value: 92.00},
{Time: 14, Value: 105.70},
{Time: 15, Value: 122.80},
{Time: 16, Value: 131.70},
{Time: 17, Value: 151.30},
{Time: 18, Value: 179.30},
{Time: 19, Value: 203.20},
}
hw := query.NewFloatHoltWintersReducer(10, 0, true, 1)
for _, p := range series {
hw.AggregateFloat(&p)
}
points := hw.Emit()
forecasted := []query.FloatPoint{
{Time: 1, Value: 3.93},
{Time: 2, Value: 4.957405463559748},
{Time: 3, Value: 7.012210102535647},
{Time: 4, Value: 10.099589257439924},
{Time: 5, Value: 14.229926188104242},
{Time: 6, Value: 19.418878968703797},
{Time: 7, Value: 25.68749172281409},
{Time: 8, Value: 33.062351305731305},
{Time: 9, Value: 41.575791076125206},
{Time: 10, Value: 51.26614395589263},
{Time: 11, Value: 62.178047564264595},
{Time: 12, Value: 74.36280483872488},
{Time: 13, Value: 87.87880423073163},
{Time: 14, Value: 102.79200429905801},
{Time: 15, Value: 119.17648832929542},
{Time: 16, Value: 137.11509549747296},
{Time: 17, Value: 156.70013608313175},
{Time: 18, Value: 178.03419933863566},
{Time: 19, Value: 201.23106385518594},
{Time: 20, Value: 226.4167216525905},
{Time: 21, Value: 253.73052878285205},
{Time: 22, Value: 283.32649700397553},
{Time: 23, Value: 315.37474308085984},
{Time: 24, Value: 350.06311454009256},
{Time: 25, Value: 387.59901328556873},
{Time: 26, Value: 428.21144141893404},
{Time: 27, Value: 472.1532969569147},
{Time: 28, Value: 519.7039509590035},
{Time: 29, Value: 571.1721419458248},
}
if exp, got := len(forecasted), len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
for i := range forecasted {
if exp, got := forecasted[i].Time, points[i].Time; got != exp {
t.Errorf("unexpected time on points[%d] got %v exp %v", i, got, exp)
}
if exp, got := forecasted[i].Value, points[i].Value; !almostEqual(got, exp) {
t.Errorf("unexpected value on points[%d] got %v exp %v", i, got, exp)
}
}
}
func TestHoltWinters_USPopulation_Missing(t *testing.T) {
series := []query.FloatPoint{
{Time: 1, Value: 3.93},
{Time: 2, Value: 5.31},
{Time: 3, Value: 7.24},
{Time: 4, Value: 9.64},
{Time: 5, Value: 12.90},
{Time: 6, Value: 17.10},
{Time: 7, Value: 23.20},
{Time: 8, Value: 31.40},
{Time: 10, Value: 50.20},
{Time: 11, Value: 62.90},
{Time: 12, Value: 76.00},
{Time: 13, Value: 92.00},
{Time: 15, Value: 122.80},
{Time: 16, Value: 131.70},
{Time: 17, Value: 151.30},
{Time: 19, Value: 203.20},
}
hw := query.NewFloatHoltWintersReducer(10, 0, true, 1)
for _, p := range series {
hw.AggregateFloat(&p)
}
points := hw.Emit()
forecasted := []query.FloatPoint{
{Time: 1, Value: 3.93},
{Time: 2, Value: 4.8931364428135105},
{Time: 3, Value: 6.962653629047061},
{Time: 4, Value: 10.056207765903274},
{Time: 5, Value: 14.18435088129532},
{Time: 6, Value: 19.362939306110846},
{Time: 7, Value: 25.613247940326584},
{Time: 8, Value: 32.96213087008264},
{Time: 9, Value: 41.442230043017204},
{Time: 10, Value: 51.09223428526052},
{Time: 11, Value: 61.95719155158485},
{Time: 12, Value: 74.08887794968567},
{Time: 13, Value: 87.54622778052787},
{Time: 14, Value: 102.39582960014131},
{Time: 15, Value: 118.7124941463221},
{Time: 16, Value: 136.57990089987464},
{Time: 17, Value: 156.09133107941278},
{Time: 18, Value: 177.35049601833734},
{Time: 19, Value: 200.472471161683},
{Time: 20, Value: 225.58474737097785},
{Time: 21, Value: 252.82841286206823},
{Time: 22, Value: 282.35948095261017},
{Time: 23, Value: 314.3503808953992},
{Time: 24, Value: 348.99163145856954},
{Time: 25, Value: 386.49371962730555},
{Time: 26, Value: 427.08920989407727},
{Time: 27, Value: 471.0351131332573},
{Time: 28, Value: 518.615548088049},
{Time: 29, Value: 570.1447331101863},
}
if exp, got := len(forecasted), len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
for i := range forecasted {
if exp, got := forecasted[i].Time, points[i].Time; got != exp {
t.Errorf("unexpected time on points[%d] got %v exp %v", i, got, exp)
}
if exp, got := forecasted[i].Value, points[i].Value; !almostEqual(got, exp) {
t.Errorf("unexpected value on points[%d] got %v exp %v", i, got, exp)
}
}
}
func TestHoltWinters_RoundTime(t *testing.T) {
maxTime := time.Unix(0, influxql.MaxTime).Round(time.Second).UnixNano()
data := []query.FloatPoint{
{Time: maxTime - int64(5*time.Second), Value: 1},
{Time: maxTime - int64(4*time.Second+103*time.Millisecond), Value: 10},
{Time: maxTime - int64(3*time.Second+223*time.Millisecond), Value: 2},
{Time: maxTime - int64(2*time.Second+481*time.Millisecond), Value: 11},
}
hw := query.NewFloatHoltWintersReducer(2, 2, true, time.Second)
for _, p := range data {
hw.AggregateFloat(&p)
}
points := hw.Emit()
forecasted := []query.FloatPoint{
{Time: maxTime - int64(5*time.Second), Value: 1},
{Time: maxTime - int64(4*time.Second), Value: 10.006729104838234},
{Time: maxTime - int64(3*time.Second), Value: 1.998341814469269},
{Time: maxTime - int64(2*time.Second), Value: 10.997858830631172},
{Time: maxTime - int64(1*time.Second), Value: 4.085860238030013},
{Time: maxTime - int64(0*time.Second), Value: 11.35713604403339},
}
if exp, got := len(forecasted), len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
for i := range forecasted {
if exp, got := forecasted[i].Time, points[i].Time; got != exp {
t.Errorf("unexpected time on points[%d] got %v exp %v", i, got, exp)
}
if exp, got := forecasted[i].Value, points[i].Value; !almostEqual(got, exp) {
t.Errorf("unexpected value on points[%d] got %v exp %v", i, got, exp)
}
}
}
func TestHoltWinters_MaxTime(t *testing.T) {
data := []query.FloatPoint{
{Time: influxql.MaxTime - 1, Value: 1},
{Time: influxql.MaxTime, Value: 2},
}
hw := query.NewFloatHoltWintersReducer(1, 0, true, 1)
for _, p := range data {
hw.AggregateFloat(&p)
}
points := hw.Emit()
forecasted := []query.FloatPoint{
{Time: influxql.MaxTime - 1, Value: 1},
{Time: influxql.MaxTime, Value: 2.001516944066403},
{Time: influxql.MaxTime + 1, Value: 2.5365248972488343},
}
if exp, got := len(forecasted), len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
for i := range forecasted {
if exp, got := forecasted[i].Time, points[i].Time; got != exp {
t.Errorf("unexpected time on points[%d] got %v exp %v", i, got, exp)
}
if exp, got := forecasted[i].Value, points[i].Value; !almostEqual(got, exp) {
t.Errorf("unexpected value on points[%d] got %v exp %v", i, got, exp)
}
}
}
// TestSample_AllSamplesSeen attempts to verify that it is possible
// to get every subsample in a reasonable number of iterations.
//
// The idea here is that 30 iterations should be enough to hit every possible
// sequence at least once.
func TestSample_AllSamplesSeen(t *testing.T) {
ps := []query.FloatPoint{
{Time: 1, Value: 1},
{Time: 2, Value: 2},
{Time: 3, Value: 3},
}
// List of all the possible subsamples
samples := [][]query.FloatPoint{
{
{Time: 1, Value: 1},
{Time: 2, Value: 2},
},
{
{Time: 1, Value: 1},
{Time: 3, Value: 3},
},
{
{Time: 2, Value: 2},
{Time: 3, Value: 3},
},
}
// 30 iterations should be sufficient to guarantee that
// we hit every possible subsample.
for i := 0; i < 30; i++ {
s := query.NewFloatSampleReducer(2)
for _, p := range ps {
s.AggregateFloat(&p)
}
points := s.Emit()
for i, sample := range samples {
// if we find a sample that it matches, remove it from
// this list of possible samples
if deep.Equal(sample, points) {
samples = append(samples[:i], samples[i+1:]...)
break
}
}
// if samples is empty we've seen every sample, so we're done
if len(samples) == 0 {
return
}
// The FloatSampleReducer is seeded with time.Now().UnixNano(), and without this sleep,
// this test will fail on machines where UnixNano doesn't return full resolution.
// Specifically, some Windows machines will only return timestamps accurate to 100ns.
// While iterating through this test without an explicit sleep,
// we would only see one or two unique seeds across all the calls to NewFloatSampleReducer.
time.Sleep(time.Millisecond)
}
// If we missed a sample, report the error
if len(samples) != 0 {
t.Fatalf("expected all samples to be seen; unseen samples: %#v", samples)
}
}
func TestSample_SampleSizeLessThanNumPoints(t *testing.T) {
s := query.NewFloatSampleReducer(2)
ps := []query.FloatPoint{
{Time: 1, Value: 1},
{Time: 2, Value: 2},
{Time: 3, Value: 3},
}
for _, p := range ps {
s.AggregateFloat(&p)
}
points := s.Emit()
if exp, got := 2, len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
}
func TestSample_SampleSizeGreaterThanNumPoints(t *testing.T) {
s := query.NewFloatSampleReducer(4)
ps := []query.FloatPoint{
{Time: 1, Value: 1},
{Time: 2, Value: 2},
{Time: 3, Value: 3},
}
for _, p := range ps {
s.AggregateFloat(&p)
}
points := s.Emit()
if exp, got := len(ps), len(points); exp != got {
t.Fatalf("unexpected number of points emitted: got %d exp %d", got, exp)
}
if !deep.Equal(ps, points) {
t.Fatalf("unexpected points: %s", spew.Sdump(points))
}
}

3
influxql/query/internal/gota/README.md Normal file
View File

@ -0,0 +1,3 @@
This is a port of [gota](https://github.com/phemmer/gota) to be adapted inside of InfluxDB.
This port was made with the permission of the author, Patrick Hemmer, and has been modified to remove dependencies that are not part of InfluxDB.

127
influxql/query/internal/gota/cmo.go Normal file
View File

@ -0,0 +1,127 @@
package gota
// CMO - Chande Momentum Oscillator (https://www.fidelity.com/learning-center/trading-investing/technical-analysis/technical-indicator-guide/cmo)
type CMO struct {
points []cmoPoint
sumUp float64
sumDown float64
count int
idx int // index of newest point
}
type cmoPoint struct {
price float64
diff float64
}
// NewCMO constructs a new CMO.
func NewCMO(inTimePeriod int) *CMO {
return &CMO{
points: make([]cmoPoint, inTimePeriod-1),
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (cmo *CMO) WarmCount() int {
return len(cmo.points)
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (cmo *CMO) Add(v float64) float64 {
idxOldest := cmo.idx + 1
if idxOldest == len(cmo.points) {
idxOldest = 0
}
var diff float64
if cmo.count != 0 {
prev := cmo.points[cmo.idx]
diff = v - prev.price
if diff > 0 {
cmo.sumUp += diff
} else if diff < 0 {
cmo.sumDown -= diff
}
}
var outV float64
if cmo.sumUp != 0 || cmo.sumDown != 0 {
outV = 100.0 * ((cmo.sumUp - cmo.sumDown) / (cmo.sumUp + cmo.sumDown))
}
oldest := cmo.points[idxOldest]
//NOTE: because we're just adding and subtracting the difference, and not recalculating sumUp/sumDown using cmo.points[].price, it's possible for imprecision to creep in over time. Not sure how significant this is going to be, but if we want to fix it, we could recalculate it from scratch every N points.
if oldest.diff > 0 {
cmo.sumUp -= oldest.diff
} else if oldest.diff < 0 {
cmo.sumDown += oldest.diff
}
p := cmoPoint{
price: v,
diff: diff,
}
cmo.points[idxOldest] = p
cmo.idx = idxOldest
if !cmo.Warmed() {
cmo.count++
}
return outV
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (cmo *CMO) Warmed() bool {
return cmo.count == len(cmo.points)+2
}
// CMOS is a smoothed version of the Chande Momentum Oscillator.
// This is the version of CMO utilized by ta-lib.
type CMOS struct {
emaUp EMA
emaDown EMA
lastV float64
}
// NewCMOS constructs a new CMOS.
func NewCMOS(inTimePeriod int, warmType WarmupType) *CMOS {
ema := NewEMA(inTimePeriod+1, warmType)
ema.alpha = float64(1) / float64(inTimePeriod)
return &CMOS{
emaUp: *ema,
emaDown: *ema,
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (cmos CMOS) WarmCount() int {
return cmos.emaUp.WarmCount()
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (cmos CMOS) Warmed() bool {
return cmos.emaUp.Warmed()
}
// Last returns the last output value.
func (cmos CMOS) Last() float64 {
up := cmos.emaUp.Last()
down := cmos.emaDown.Last()
return 100.0 * ((up - down) / (up + down))
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (cmos *CMOS) Add(v float64) float64 {
var up float64
var down float64
if v > cmos.lastV {
up = v - cmos.lastV
} else if v < cmos.lastV {
down = cmos.lastV - v
}
cmos.emaUp.Add(up)
cmos.emaDown.Add(down)
cmos.lastV = v
return cmos.Last()
}

41
influxql/query/internal/gota/cmo_test.go Normal file
View File

@ -0,0 +1,41 @@
package gota
import "testing"
func TestCMO(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
expList := []float64{100, 100, 100, 100, 100, 80, 60, 40, 20, 0, -20, -40, -60, -80, -100, -100, -100, -100, -100}
cmo := NewCMO(10)
var actList []float64
for _, v := range list {
if vOut := cmo.Add(v); cmo.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 1e-7); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}
func TestCMOS(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
// expList is generated by the following code:
// expList, _ := talib.Cmo(list, 10, nil)
expList := []float64{100, 100, 100, 100, 100, 80, 61.999999999999986, 45.79999999999999, 31.22, 18.097999999999992, 6.288199999999988, -4.340620000000012, -13.906558000000008, -22.515902200000014, -30.264311980000013, -37.23788078200001, -43.51409270380002, -49.16268343342002, -54.24641509007802}
cmo := NewCMOS(10, WarmSMA)
var actList []float64
for _, v := range list {
if vOut := cmo.Add(v); cmo.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 1e-7); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}

188
influxql/query/internal/gota/ema.go Normal file
View File

@ -0,0 +1,188 @@
package gota
import (
"fmt"
)
type AlgSimple interface {
Add(float64) float64
Warmed() bool
WarmCount() int
}
type WarmupType int8
const (
WarmEMA WarmupType = iota // Exponential Moving Average
WarmSMA // Simple Moving Average
)
func ParseWarmupType(wt string) (WarmupType, error) {
switch wt {
case "exponential":
return WarmEMA, nil
case "simple":
return WarmSMA, nil
default:
return 0, fmt.Errorf("invalid warmup type '%s'", wt)
}
}
// EMA - Exponential Moving Average (http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:moving_averages#exponential_moving_average_calculation)
type EMA struct {
inTimePeriod int
last float64
count int
alpha float64
warmType WarmupType
}
// NewEMA constructs a new EMA.
//
// When warmed with WarmSMA the first inTimePeriod samples will result in a simple average, switching to exponential moving average after warmup is complete.
//
// When warmed with WarmEMA the algorithm immediately starts using an exponential moving average for the output values. During the warmup period the alpha value is scaled to prevent unbalanced weighting on initial values.
func NewEMA(inTimePeriod int, warmType WarmupType) *EMA {
return &EMA{
inTimePeriod: inTimePeriod,
alpha: 2 / float64(inTimePeriod+1),
warmType: warmType,
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (ema *EMA) WarmCount() int {
return ema.inTimePeriod - 1
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (ema *EMA) Warmed() bool {
return ema.count == ema.inTimePeriod
}
// Last returns the last output value.
func (ema *EMA) Last() float64 {
return ema.last
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (ema *EMA) Add(v float64) float64 {
var avg float64
if ema.count == 0 {
avg = v
} else {
lastAvg := ema.Last()
if !ema.Warmed() {
if ema.warmType == WarmSMA {
avg = (lastAvg*float64(ema.count) + v) / float64(ema.count+1)
} else { // ema.warmType == WarmEMA
// scale the alpha so that we don't excessively weight the result towards the first value
alpha := 2 / float64(ema.count+2)
avg = (v-lastAvg)*alpha + lastAvg
}
} else {
avg = (v-lastAvg)*ema.alpha + lastAvg
}
}
ema.last = avg
if ema.count < ema.inTimePeriod {
// don't just keep incrementing to prevent potential overflow
ema.count++
}
return avg
}
// DEMA - Double Exponential Moving Average (https://en.wikipedia.org/wiki/Double_exponential_moving_average)
type DEMA struct {
ema1 EMA
ema2 EMA
}
// NewDEMA constructs a new DEMA.
//
// When warmed with WarmSMA the first inTimePeriod samples will result in a simple average, switching to exponential moving average after warmup is complete.
//
// When warmed with WarmEMA the algorithm immediately starts using an exponential moving average for the output values. During the warmup period the alpha value is scaled to prevent unbalanced weighting on initial values.
func NewDEMA(inTimePeriod int, warmType WarmupType) *DEMA {
return &DEMA{
ema1: *NewEMA(inTimePeriod, warmType),
ema2: *NewEMA(inTimePeriod, warmType),
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (dema *DEMA) WarmCount() int {
if dema.ema1.warmType == WarmEMA {
return dema.ema1.WarmCount()
}
return dema.ema1.WarmCount() + dema.ema2.WarmCount()
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (dema *DEMA) Add(v float64) float64 {
avg1 := dema.ema1.Add(v)
var avg2 float64
if dema.ema1.Warmed() || dema.ema1.warmType == WarmEMA {
avg2 = dema.ema2.Add(avg1)
} else {
avg2 = avg1
}
return 2*avg1 - avg2
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (dema *DEMA) Warmed() bool {
return dema.ema2.Warmed()
}
// TEMA - Triple Exponential Moving Average (https://en.wikipedia.org/wiki/Triple_exponential_moving_average)
type TEMA struct {
ema1 EMA
ema2 EMA
ema3 EMA
}
// NewTEMA constructs a new TEMA.
//
// When warmed with WarmSMA the first inTimePeriod samples will result in a simple average, switching to exponential moving average after warmup is complete.
//
// When warmed with WarmEMA the algorithm immediately starts using an exponential moving average for the output values. During the warmup period the alpha value is scaled to prevent unbalanced weighting on initial values.
func NewTEMA(inTimePeriod int, warmType WarmupType) *TEMA {
return &TEMA{
ema1: *NewEMA(inTimePeriod, warmType),
ema2: *NewEMA(inTimePeriod, warmType),
ema3: *NewEMA(inTimePeriod, warmType),
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (tema *TEMA) WarmCount() int {
if tema.ema1.warmType == WarmEMA {
return tema.ema1.WarmCount()
}
return tema.ema1.WarmCount() + tema.ema2.WarmCount() + tema.ema3.WarmCount()
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (tema *TEMA) Add(v float64) float64 {
avg1 := tema.ema1.Add(v)
var avg2 float64
if tema.ema1.Warmed() || tema.ema1.warmType == WarmEMA {
avg2 = tema.ema2.Add(avg1)
} else {
avg2 = avg1
}
var avg3 float64
if tema.ema2.Warmed() || tema.ema2.warmType == WarmEMA {
avg3 = tema.ema3.Add(avg2)
} else {
avg3 = avg2
}
return 3*avg1 - 3*avg2 + avg3
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (tema *TEMA) Warmed() bool {
return tema.ema3.Warmed()
}

114
influxql/query/internal/gota/ema_test.go Normal file
View File

@ -0,0 +1,114 @@
package gota
import "testing"
func TestEMA(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
// expList is generated by the following code:
// expList, _ := talib.Ema(list, 10, nil)
expList := []float64{5.5, 6.5, 7.5, 8.5, 9.5, 10.5, 11.136363636363637, 11.475206611570249, 11.570623591284749, 11.466873847414794, 11.200169511521196, 10.800138691244614, 10.291022565563775, 9.692654826370362, 9.021263039757569, 8.290124305256192, 7.510101704300521, 6.690083212609517, 5.837340810316878, 4.957824299350173}
ema := NewEMA(10, WarmSMA)
var actList []float64
for _, v := range list {
if vOut := ema.Add(v); ema.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 0.0000001); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}
func TestDEMA(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
// expList is generated by the following code:
// expList, _ := talib.Dema(list, 10, nil)
expList := []float64{13.568840926166246, 12.701748119313985, 11.701405062848783, 10.611872766773773, 9.465595022565749, 8.28616628396151, 7.090477085921927, 5.8903718513360275, 4.693925476073202, 3.5064225149113692, 2.331104912318361}
dema := NewDEMA(10, WarmSMA)
var actList []float64
for _, v := range list {
if vOut := dema.Add(v); dema.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 0.0000001); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}
func TestTEMA(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
// expList is generated by the following code:
// expList, _ := talib.Tema(list, 4, nil)
expList := []float64{10, 11, 12, 13, 14, 15, 14.431999999999995, 13.345600000000001, 12.155520000000001, 11, 9.906687999999997, 8.86563072, 7.8589122560000035, 6.871005491200005, 5.891160883200005, 4.912928706560004, 3.932955104051203, 2.9498469349785603, 1.9633255712030717, 0.9736696408637435}
tema := NewTEMA(4, WarmSMA)
var actList []float64
for _, v := range list {
if vOut := tema.Add(v); tema.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 0.0000001); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}
func TestEmaWarmCount(t *testing.T) {
period := 9
ema := NewEMA(period, WarmSMA)
var i int
for i = 0; i < period*10; i++ {
ema.Add(float64(i))
if ema.Warmed() {
break
}
}
if got, want := i, ema.WarmCount(); got != want {
t.Errorf("unexpected warm count: got=%d want=%d", got, want)
}
}
func TestDemaWarmCount(t *testing.T) {
period := 9
dema := NewDEMA(period, WarmSMA)
var i int
for i = 0; i < period*10; i++ {
dema.Add(float64(i))
if dema.Warmed() {
break
}
}
if got, want := i, dema.WarmCount(); got != want {
t.Errorf("unexpected warm count: got=%d want=%d", got, want)
}
}
func TestTemaWarmCount(t *testing.T) {
period := 9
tema := NewTEMA(period, WarmSMA)
var i int
for i = 0; i < period*10; i++ {
tema.Add(float64(i))
if tema.Warmed() {
break
}
}
if got, want := i, tema.WarmCount(); got != want {
t.Errorf("unexpected warm count: got=%d want=%d", got, want)
}
}

113
influxql/query/internal/gota/kama.go Normal file
View File

@ -0,0 +1,113 @@
package gota
import (
"math"
)
// KER - Kaufman's Efficiency Ratio (http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:kaufman_s_adaptive_moving_average#efficiency_ratio_er)
type KER struct {
points []kerPoint
noise float64
count int
idx int // index of newest point
}
type kerPoint struct {
price float64
diff float64
}
// NewKER constructs a new KER.
func NewKER(inTimePeriod int) *KER {
return &KER{
points: make([]kerPoint, inTimePeriod),
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (ker *KER) WarmCount() int {
return len(ker.points)
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (ker *KER) Add(v float64) float64 {
//TODO this does not return a sensible value if not warmed.
n := len(ker.points)
idxOldest := ker.idx + 1
if idxOldest >= n {
idxOldest = 0
}
signal := math.Abs(v - ker.points[idxOldest].price)
kp := kerPoint{
price: v,
diff: math.Abs(v - ker.points[ker.idx].price),
}
ker.noise -= ker.points[idxOldest].diff
ker.noise += kp.diff
noise := ker.noise
ker.idx = idxOldest
ker.points[ker.idx] = kp
if !ker.Warmed() {
ker.count++
}
if signal == 0 || noise == 0 {
return 0
}
return signal / noise
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (ker *KER) Warmed() bool {
return ker.count == len(ker.points)+1
}
// KAMA - Kaufman's Adaptive Moving Average (http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:kaufman_s_adaptive_moving_average)
type KAMA struct {
ker KER
last float64
}
// NewKAMA constructs a new KAMA.
func NewKAMA(inTimePeriod int) *KAMA {
ker := NewKER(inTimePeriod)
return &KAMA{
ker: *ker,
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (kama *KAMA) WarmCount() int {
return kama.ker.WarmCount()
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (kama *KAMA) Add(v float64) float64 {
if !kama.Warmed() {
/*
// initialize with a simple moving average
kama.last = 0
for _, v := range kama.ker.points[:kama.ker.count] {
kama.last += v
}
kama.last /= float64(kama.ker.count + 1)
*/
// initialize with the last value
kama.last = kama.ker.points[kama.ker.idx].price
}
er := kama.ker.Add(v)
sc := math.Pow(er*(2.0/(2.0+1.0)-2.0/(30.0+1.0))+2.0/(30.0+1.0), 2)
kama.last = kama.last + sc*(v-kama.last)
return kama.last
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (kama *KAMA) Warmed() bool {
return kama.ker.Warmed()
}

70
influxql/query/internal/gota/kama_test.go Normal file
View File

@ -0,0 +1,70 @@
package gota
import "testing"
func TestKER(t *testing.T) {
list := []float64{20, 21, 22, 23, 22, 21}
expList := []float64{1, 1.0 / 3, 1.0 / 3}
ker := NewKER(3)
var actList []float64
for _, v := range list {
if vOut := ker.Add(v); ker.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 0.0000001); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}
func TestKAMA(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
// expList is generated by the following code:
// expList, _ := talib.Cmo(list, 10, nil)
expList := []float64{10.444444444444445, 11.135802469135802, 11.964334705075446, 12.869074836153025, 13.81615268675168, 13.871008014588556, 13.71308456353558, 13.553331356741122, 13.46599437575161, 13.4515677602438, 13.29930139347417, 12.805116570729284, 11.752584300922967, 10.036160535131103, 7.797866963961725, 6.109926091089847, 4.727736717272138, 3.5154092873734104, 2.3974496040963396}
kama := NewKAMA(10)
var actList []float64
for _, v := range list {
if vOut := kama.Add(v); kama.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 0.0000001); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}
func TestKAMAWarmCount(t *testing.T) {
period := 9
kama := NewKAMA(period)
var i int
for i = 0; i < period*10; i++ {
kama.Add(float64(i))
if kama.Warmed() {
break
}
}
if got, want := i, kama.WarmCount(); got != want {
t.Errorf("unexpected warm count: got=%d want=%d", got, want)
}
}
var BenchmarkKAMAVal float64
func BenchmarkKAMA(b *testing.B) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
for n := 0; n < b.N; n++ {
kama := NewKAMA(5)
for _, v := range list {
BenchmarkKAMAVal = kama.Add(v)
}
}
}

48
influxql/query/internal/gota/rsi.go Normal file
View File

@ -0,0 +1,48 @@
package gota
// RSI - Relative Strength Index (http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:relative_strength_index_rsi)
type RSI struct {
emaUp EMA
emaDown EMA
lastV float64
}
// NewRSI constructs a new RSI.
func NewRSI(inTimePeriod int, warmType WarmupType) *RSI {
ema := NewEMA(inTimePeriod+1, warmType)
ema.alpha = float64(1) / float64(inTimePeriod)
return &RSI{
emaUp: *ema,
emaDown: *ema,
}
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (rsi RSI) WarmCount() int {
return rsi.emaUp.WarmCount()
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (rsi RSI) Warmed() bool {
return rsi.emaUp.Warmed()
}
// Last returns the last output value.
func (rsi RSI) Last() float64 {
return 100 - (100 / (1 + rsi.emaUp.Last()/rsi.emaDown.Last()))
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (rsi *RSI) Add(v float64) float64 {
var up float64
var down float64
if v > rsi.lastV {
up = v - rsi.lastV
} else if v < rsi.lastV {
down = rsi.lastV - v
}
rsi.emaUp.Add(up)
rsi.emaDown.Add(down)
rsi.lastV = v
return rsi.Last()
}

23
influxql/query/internal/gota/rsi_test.go Normal file
View File

@ -0,0 +1,23 @@
package gota
import "testing"
func TestRSI(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
// expList is generated by the following code:
// expList, _ := talib.Rsi(list, 10, nil)
expList := []float64{100, 100, 100, 100, 100, 90, 81, 72.89999999999999, 65.61, 59.04899999999999, 53.144099999999995, 47.82969, 43.04672099999999, 38.74204889999999, 34.86784400999999, 31.381059608999994, 28.242953648099995, 25.418658283289997, 22.876792454961}
rsi := NewRSI(10, WarmSMA)
var actList []float64
for _, v := range list {
if vOut := rsi.Add(v); rsi.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 0.0000001); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}

53
influxql/query/internal/gota/trix.go Normal file
View File

@ -0,0 +1,53 @@
package gota
// Trix - TRIple Exponential average (http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:trix)
type TRIX struct {
ema1 EMA
ema2 EMA
ema3 EMA
last float64
count int
}
// NewTRIX constructs a new TRIX.
func NewTRIX(inTimePeriod int, warmType WarmupType) *TRIX {
ema1 := NewEMA(inTimePeriod, warmType)
ema2 := NewEMA(inTimePeriod, warmType)
ema3 := NewEMA(inTimePeriod, warmType)
return &TRIX{
ema1: *ema1,
ema2: *ema2,
ema3: *ema3,
}
}
// Add adds a new sample value to the algorithm and returns the computed value.
func (trix *TRIX) Add(v float64) float64 {
cur := trix.ema1.Add(v)
if trix.ema1.Warmed() || trix.ema1.warmType == WarmEMA {
cur = trix.ema2.Add(cur)
if trix.ema2.Warmed() || trix.ema2.warmType == WarmEMA {
cur = trix.ema3.Add(cur)
}
}
rate := ((cur / trix.last) - 1) * 100
trix.last = cur
if !trix.Warmed() && trix.ema3.Warmed() {
trix.count++
}
return rate
}
// WarmCount returns the number of samples that must be provided for the algorithm to be fully "warmed".
func (trix *TRIX) WarmCount() int {
if trix.ema1.warmType == WarmEMA {
return trix.ema1.WarmCount() + 1
}
return trix.ema1.WarmCount()*3 + 1
}
// Warmed indicates whether the algorithm has enough data to generate accurate results.
func (trix *TRIX) Warmed() bool {
return trix.count == 2
}

23
influxql/query/internal/gota/trix_test.go Normal file
View File

@ -0,0 +1,23 @@
package gota
import "testing"
func TestTRIX(t *testing.T) {
list := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
// expList is generated by the following code:
// expList, _ := talib.Trix(list, 4, nil)
expList := []float64{18.181818181818187, 15.384615384615374, 13.33333333333333, 11.764705882352944, 10.526315789473696, 8.304761904761904, 5.641927541329594, 3.0392222148232007, 0.7160675740302658, -1.2848911076603242, -2.9999661985600667, -4.493448741755901, -5.836238000516913, -7.099092024379772, -8.352897627933453, -9.673028502435233, -11.147601363985949, -12.891818138458877, -15.074463280730022}
trix := NewTRIX(4, WarmSMA)
var actList []float64
for _, v := range list {
if vOut := trix.Add(v); trix.Warmed() {
actList = append(actList, vOut)
}
}
if diff := diffFloats(expList, actList, 1e-7); diff != "" {
t.Errorf("unexpected floats:\n%s", diff)
}
}

10
influxql/query/internal/gota/utils_test.go Normal file
View File

@ -0,0 +1,10 @@
package gota
import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
)
func diffFloats(exp, act []float64, delta float64) string {
return cmp.Diff(exp, act, cmpopts.EquateApprox(0, delta))
}

606
influxql/query/internal/internal.pb.go Normal file
View File

@ -0,0 +1,606 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: internal/internal.proto
/*
Package query is a generated protocol buffer package.
It is generated from these files:
internal/internal.proto
It has these top-level messages:
Point
Aux
IteratorOptions
Measurements
Measurement
Interval
IteratorStats
VarRef
*/
package query
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
type Point struct {
Name *string `protobuf:"bytes,1,req,name=Name" json:"Name,omitempty"`
Tags *string `protobuf:"bytes,2,req,name=Tags" json:"Tags,omitempty"`
Time *int64 `protobuf:"varint,3,req,name=Time" json:"Time,omitempty"`
Nil *bool `protobuf:"varint,4,req,name=Nil" json:"Nil,omitempty"`
Aux []*Aux `protobuf:"bytes,5,rep,name=Aux" json:"Aux,omitempty"`
Aggregated *uint32 `protobuf:"varint,6,opt,name=Aggregated" json:"Aggregated,omitempty"`
FloatValue *float64 `protobuf:"fixed64,7,opt,name=FloatValue" json:"FloatValue,omitempty"`
IntegerValue *int64 `protobuf:"varint,8,opt,name=IntegerValue" json:"IntegerValue,omitempty"`
StringValue *string `protobuf:"bytes,9,opt,name=StringValue" json:"StringValue,omitempty"`
BooleanValue *bool `protobuf:"varint,10,opt,name=BooleanValue" json:"BooleanValue,omitempty"`
UnsignedValue *uint64 `protobuf:"varint,12,opt,name=UnsignedValue" json:"UnsignedValue,omitempty"`
Stats *IteratorStats `protobuf:"bytes,11,opt,name=Stats" json:"Stats,omitempty"`
Trace []byte `protobuf:"bytes,13,opt,name=Trace" json:"Trace,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Point) Reset() { *m = Point{} }
func (m *Point) String() string { return proto.CompactTextString(m) }
func (*Point) ProtoMessage() {}
func (*Point) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} }
func (m *Point) GetName() string {
if m != nil && m.Name != nil {
return *m.Name
}
return ""
}
func (m *Point) GetTags() string {
if m != nil && m.Tags != nil {
return *m.Tags
}
return ""
}
func (m *Point) GetTime() int64 {
if m != nil && m.Time != nil {
return *m.Time
}
return 0
}
func (m *Point) GetNil() bool {
if m != nil && m.Nil != nil {
return *m.Nil
}
return false
}
func (m *Point) GetAux() []*Aux {
if m != nil {
return m.Aux
}
return nil
}
func (m *Point) GetAggregated() uint32 {
if m != nil && m.Aggregated != nil {
return *m.Aggregated
}
return 0
}
func (m *Point) GetFloatValue() float64 {
if m != nil && m.FloatValue != nil {
return *m.FloatValue
}
return 0
}
func (m *Point) GetIntegerValue() int64 {
if m != nil && m.IntegerValue != nil {
return *m.IntegerValue
}
return 0
}
func (m *Point) GetStringValue() string {
if m != nil && m.StringValue != nil {
return *m.StringValue
}
return ""
}
func (m *Point) GetBooleanValue() bool {
if m != nil && m.BooleanValue != nil {
return *m.BooleanValue
}
return false
}
func (m *Point) GetUnsignedValue() uint64 {
if m != nil && m.UnsignedValue != nil {
return *m.UnsignedValue
}
return 0
}
func (m *Point) GetStats() *IteratorStats {
if m != nil {
return m.Stats
}
return nil
}
func (m *Point) GetTrace() []byte {
if m != nil {
return m.Trace
}
return nil
}
type Aux struct {
DataType *int32 `protobuf:"varint,1,req,name=DataType" json:"DataType,omitempty"`
FloatValue *float64 `protobuf:"fixed64,2,opt,name=FloatValue" json:"FloatValue,omitempty"`
IntegerValue *int64 `protobuf:"varint,3,opt,name=IntegerValue" json:"IntegerValue,omitempty"`
StringValue *string `protobuf:"bytes,4,opt,name=StringValue" json:"StringValue,omitempty"`
BooleanValue *bool `protobuf:"varint,5,opt,name=BooleanValue" json:"BooleanValue,omitempty"`
UnsignedValue *uint64 `protobuf:"varint,6,opt,name=UnsignedValue" json:"UnsignedValue,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Aux) Reset() { *m = Aux{} }
func (m *Aux) String() string { return proto.CompactTextString(m) }
func (*Aux) ProtoMessage() {}
func (*Aux) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} }
func (m *Aux) GetDataType() int32 {
if m != nil && m.DataType != nil {
return *m.DataType
}
return 0
}
func (m *Aux) GetFloatValue() float64 {
if m != nil && m.FloatValue != nil {
return *m.FloatValue
}
return 0
}
func (m *Aux) GetIntegerValue() int64 {
if m != nil && m.IntegerValue != nil {
return *m.IntegerValue
}
return 0
}
func (m *Aux) GetStringValue() string {
if m != nil && m.StringValue != nil {
return *m.StringValue
}
return ""
}
func (m *Aux) GetBooleanValue() bool {
if m != nil && m.BooleanValue != nil {
return *m.BooleanValue
}
return false
}
func (m *Aux) GetUnsignedValue() uint64 {
if m != nil && m.UnsignedValue != nil {
return *m.UnsignedValue
}
return 0
}
type IteratorOptions struct {
Expr *string `protobuf:"bytes,1,opt,name=Expr" json:"Expr,omitempty"`
Aux []string `protobuf:"bytes,2,rep,name=Aux" json:"Aux,omitempty"`
Fields []*VarRef `protobuf:"bytes,17,rep,name=Fields" json:"Fields,omitempty"`
Sources []*Measurement `protobuf:"bytes,3,rep,name=Sources" json:"Sources,omitempty"`
Interval *Interval `protobuf:"bytes,4,opt,name=Interval" json:"Interval,omitempty"`
Dimensions []string `protobuf:"bytes,5,rep,name=Dimensions" json:"Dimensions,omitempty"`
GroupBy []string `protobuf:"bytes,19,rep,name=GroupBy" json:"GroupBy,omitempty"`
Fill *int32 `protobuf:"varint,6,opt,name=Fill" json:"Fill,omitempty"`
FillValue *float64 `protobuf:"fixed64,7,opt,name=FillValue" json:"FillValue,omitempty"`
Condition *string `protobuf:"bytes,8,opt,name=Condition" json:"Condition,omitempty"`
StartTime *int64 `protobuf:"varint,9,opt,name=StartTime" json:"StartTime,omitempty"`
EndTime *int64 `protobuf:"varint,10,opt,name=EndTime" json:"EndTime,omitempty"`
Location *string `protobuf:"bytes,21,opt,name=Location" json:"Location,omitempty"`
Ascending *bool `protobuf:"varint,11,opt,name=Ascending" json:"Ascending,omitempty"`
Limit *int64 `protobuf:"varint,12,opt,name=Limit" json:"Limit,omitempty"`
Offset *int64 `protobuf:"varint,13,opt,name=Offset" json:"Offset,omitempty"`
SLimit *int64 `protobuf:"varint,14,opt,name=SLimit" json:"SLimit,omitempty"`
SOffset *int64 `protobuf:"varint,15,opt,name=SOffset" json:"SOffset,omitempty"`
StripName *bool `protobuf:"varint,22,opt,name=StripName" json:"StripName,omitempty"`
Dedupe *bool `protobuf:"varint,16,opt,name=Dedupe" json:"Dedupe,omitempty"`
MaxSeriesN *int64 `protobuf:"varint,18,opt,name=MaxSeriesN" json:"MaxSeriesN,omitempty"`
Ordered *bool `protobuf:"varint,20,opt,name=Ordered" json:"Ordered,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *IteratorOptions) Reset() { *m = IteratorOptions{} }
func (m *IteratorOptions) String() string { return proto.CompactTextString(m) }
func (*IteratorOptions) ProtoMessage() {}
func (*IteratorOptions) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} }
func (m *IteratorOptions) GetExpr() string {
if m != nil && m.Expr != nil {
return *m.Expr
}
return ""
}
func (m *IteratorOptions) GetAux() []string {
if m != nil {
return m.Aux
}
return nil
}
func (m *IteratorOptions) GetFields() []*VarRef {
if m != nil {
return m.Fields
}
return nil
}
func (m *IteratorOptions) GetSources() []*Measurement {
if m != nil {
return m.Sources
}
return nil
}
func (m *IteratorOptions) GetInterval() *Interval {
if m != nil {
return m.Interval
}
return nil
}
func (m *IteratorOptions) GetDimensions() []string {
if m != nil {
return m.Dimensions
}
return nil
}
func (m *IteratorOptions) GetGroupBy() []string {
if m != nil {
return m.GroupBy
}
return nil
}
func (m *IteratorOptions) GetFill() int32 {
if m != nil && m.Fill != nil {
return *m.Fill
}
return 0
}
func (m *IteratorOptions) GetFillValue() float64 {
if m != nil && m.FillValue != nil {
return *m.FillValue
}
return 0
}
func (m *IteratorOptions) GetCondition() string {
if m != nil && m.Condition != nil {
return *m.Condition
}
return ""
}
func (m *IteratorOptions) GetStartTime() int64 {
if m != nil && m.StartTime != nil {
return *m.StartTime
}
return 0
}
func (m *IteratorOptions) GetEndTime() int64 {
if m != nil && m.EndTime != nil {
return *m.EndTime
}
return 0
}
func (m *IteratorOptions) GetLocation() string {
if m != nil && m.Location != nil {
return *m.Location
}
return ""
}
func (m *IteratorOptions) GetAscending() bool {
if m != nil && m.Ascending != nil {
return *m.Ascending
}
return false
}
func (m *IteratorOptions) GetLimit() int64 {
if m != nil && m.Limit != nil {
return *m.Limit
}
return 0
}
func (m *IteratorOptions) GetOffset() int64 {
if m != nil && m.Offset != nil {
return *m.Offset
}
return 0
}
func (m *IteratorOptions) GetSLimit() int64 {
if m != nil && m.SLimit != nil {
return *m.SLimit
}
return 0
}
func (m *IteratorOptions) GetSOffset() int64 {
if m != nil && m.SOffset != nil {
return *m.SOffset
}
return 0
}
func (m *IteratorOptions) GetStripName() bool {
if m != nil && m.StripName != nil {
return *m.StripName
}
return false
}
func (m *IteratorOptions) GetDedupe() bool {
if m != nil && m.Dedupe != nil {
return *m.Dedupe
}
return false
}
func (m *IteratorOptions) GetMaxSeriesN() int64 {
if m != nil && m.MaxSeriesN != nil {
return *m.MaxSeriesN
}
return 0
}
func (m *IteratorOptions) GetOrdered() bool {
if m != nil && m.Ordered != nil {
return *m.Ordered
}
return false
}
type Measurements struct {
Items []*Measurement `protobuf:"bytes,1,rep,name=Items" json:"Items,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Measurements) Reset() { *m = Measurements{} }
func (m *Measurements) String() string { return proto.CompactTextString(m) }
func (*Measurements) ProtoMessage() {}
func (*Measurements) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} }
func (m *Measurements) GetItems() []*Measurement {
if m != nil {
return m.Items
}
return nil
}
type Measurement struct {
Database *string `protobuf:"bytes,1,opt,name=Database" json:"Database,omitempty"`
RetentionPolicy *string `protobuf:"bytes,2,opt,name=RetentionPolicy" json:"RetentionPolicy,omitempty"`
Name *string `protobuf:"bytes,3,opt,name=Name" json:"Name,omitempty"`
Regex *string `protobuf:"bytes,4,opt,name=Regex" json:"Regex,omitempty"`
IsTarget *bool `protobuf:"varint,5,opt,name=IsTarget" json:"IsTarget,omitempty"`
SystemIterator *string `protobuf:"bytes,6,opt,name=SystemIterator" json:"SystemIterator,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Measurement) Reset() { *m = Measurement{} }
func (m *Measurement) String() string { return proto.CompactTextString(m) }
func (*Measurement) ProtoMessage() {}
func (*Measurement) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} }
func (m *Measurement) GetDatabase() string {
if m != nil && m.Database != nil {
return *m.Database
}
return ""
}
func (m *Measurement) GetRetentionPolicy() string {
if m != nil && m.RetentionPolicy != nil {
return *m.RetentionPolicy
}
return ""
}
func (m *Measurement) GetName() string {
if m != nil && m.Name != nil {
return *m.Name
}
return ""
}
func (m *Measurement) GetRegex() string {
if m != nil && m.Regex != nil {
return *m.Regex
}
return ""
}
func (m *Measurement) GetIsTarget() bool {
if m != nil && m.IsTarget != nil {
return *m.IsTarget
}
return false
}
func (m *Measurement) GetSystemIterator() string {
if m != nil && m.SystemIterator != nil {
return *m.SystemIterator
}
return ""
}
type Interval struct {
Duration *int64 `protobuf:"varint,1,opt,name=Duration" json:"Duration,omitempty"`
Offset *int64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Interval) Reset() { *m = Interval{} }
func (m *Interval) String() string { return proto.CompactTextString(m) }
func (*Interval) ProtoMessage() {}
func (*Interval) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} }
func (m *Interval) GetDuration() int64 {
if m != nil && m.Duration != nil {
return *m.Duration
}
return 0
}
func (m *Interval) GetOffset() int64 {
if m != nil && m.Offset != nil {
return *m.Offset
}
return 0
}
type IteratorStats struct {
SeriesN *int64 `protobuf:"varint,1,opt,name=SeriesN" json:"SeriesN,omitempty"`
PointN *int64 `protobuf:"varint,2,opt,name=PointN" json:"PointN,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *IteratorStats) Reset() { *m = IteratorStats{} }
func (m *IteratorStats) String() string { return proto.CompactTextString(m) }
func (*IteratorStats) ProtoMessage() {}
func (*IteratorStats) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} }
func (m *IteratorStats) GetSeriesN() int64 {
if m != nil && m.SeriesN != nil {
return *m.SeriesN
}
return 0
}
func (m *IteratorStats) GetPointN() int64 {
if m != nil && m.PointN != nil {
return *m.PointN
}
return 0
}
type VarRef struct {
Val *string `protobuf:"bytes,1,req,name=Val" json:"Val,omitempty"`
Type *int32 `protobuf:"varint,2,opt,name=Type" json:"Type,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *VarRef) Reset() { *m = VarRef{} }
func (m *VarRef) String() string { return proto.CompactTextString(m) }
func (*VarRef) ProtoMessage() {}
func (*VarRef) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} }
func (m *VarRef) GetVal() string {
if m != nil && m.Val != nil {
return *m.Val
}
return ""
}
func (m *VarRef) GetType() int32 {
if m != nil && m.Type != nil {
return *m.Type
}
return 0
}
func init() {
proto.RegisterType((*Point)(nil), "query.Point")
proto.RegisterType((*Aux)(nil), "query.Aux")
proto.RegisterType((*IteratorOptions)(nil), "query.IteratorOptions")
proto.RegisterType((*Measurements)(nil), "query.Measurements")
proto.RegisterType((*Measurement)(nil), "query.Measurement")
proto.RegisterType((*Interval)(nil), "query.Interval")
proto.RegisterType((*IteratorStats)(nil), "query.IteratorStats")
proto.RegisterType((*VarRef)(nil), "query.VarRef")
}
func init() { proto.RegisterFile("internal/internal.proto", fileDescriptorInternal) }
var fileDescriptorInternal = []byte{
// 796 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x55, 0x6d, 0x6f, 0xe3, 0x44,
0x10, 0x96, 0xe3, 0x3a, 0x8d, 0x27, 0xcd, 0xf5, 0x58, 0x4a, 0x59, 0xa1, 0x13, 0xb2, 0x2c, 0x40,
0x16, 0xa0, 0x22, 0xf5, 0x13, 0x9f, 0x90, 0x72, 0xf4, 0x8a, 0x2a, 0xdd, 0xb5, 0xa7, 0x4d, 0xe9,
0xf7, 0x25, 0x9e, 0x5a, 0x2b, 0x39, 0xeb, 0xb0, 0x5e, 0xa3, 0xe4, 0x07, 0xf4, 0x87, 0xf1, 0x13,
0xf8, 0x47, 0x68, 0x67, 0xd7, 0x89, 0x53, 0x81, 0x7a, 0x9f, 0x32, 0xcf, 0x33, 0x93, 0x7d, 0x79,
0xe6, 0x99, 0x35, 0x7c, 0xa9, 0xb4, 0x45, 0xa3, 0x65, 0xfd, 0x53, 0x1f, 0x5c, 0xac, 0x4d, 0x63,
0x1b, 0x96, 0xfc, 0xd9, 0xa1, 0xd9, 0xe6, 0x4f, 0x31, 0x24, 0x1f, 0x1b, 0xa5, 0x2d, 0x63, 0x70,
0x74, 0x2b, 0x57, 0xc8, 0xa3, 0x6c, 0x54, 0xa4, 0x82, 0x62, 0xc7, 0xdd, 0xcb, 0xaa, 0xe5, 0x23,
0xcf, 0xb9, 0x98, 0x38, 0xb5, 0x42, 0x1e, 0x67, 0xa3, 0x22, 0x16, 0x14, 0xb3, 0xd7, 0x10, 0xdf,
0xaa, 0x9a, 0x1f, 0x65, 0xa3, 0x62, 0x22, 0x5c, 0xc8, 0xde, 0x40, 0x3c, 0xef, 0x36, 0x3c, 0xc9,
0xe2, 0x62, 0x7a, 0x09, 0x17, 0xb4, 0xd9, 0xc5, 0xbc, 0xdb, 0x08, 0x47, 0xb3, 0xaf, 0x01, 0xe6,
0x55, 0x65, 0xb0, 0x92, 0x16, 0x4b, 0x3e, 0xce, 0xa2, 0x62, 0x26, 0x06, 0x8c, 0xcb, 0x5f, 0xd7,
0x8d, 0xb4, 0x0f, 0xb2, 0xee, 0x90, 0x1f, 0x67, 0x51, 0x11, 0x89, 0x01, 0xc3, 0x72, 0x38, 0xb9,
0xd1, 0x16, 0x2b, 0x34, 0xbe, 0x62, 0x92, 0x45, 0x45, 0x2c, 0x0e, 0x38, 0x96, 0xc1, 0x74, 0x61,
0x8d, 0xd2, 0x95, 0x2f, 0x49, 0xb3, 0xa8, 0x48, 0xc5, 0x90, 0x72, 0xab, 0xbc, 0x6d, 0x9a, 0x1a,
0xa5, 0xf6, 0x25, 0x90, 0x45, 0xc5, 0x44, 0x1c, 0x70, 0xec, 0x1b, 0x98, 0xfd, 0xae, 0x5b, 0x55,
0x69, 0x2c, 0x7d, 0xd1, 0x49, 0x16, 0x15, 0x47, 0xe2, 0x90, 0x64, 0xdf, 0x43, 0xb2, 0xb0, 0xd2,
0xb6, 0x7c, 0x9a, 0x45, 0xc5, 0xf4, 0xf2, 0x2c, 0xdc, 0xf7, 0xc6, 0xa2, 0x91, 0xb6, 0x31, 0x94,
0x13, 0xbe, 0x84, 0x9d, 0x41, 0x72, 0x6f, 0xe4, 0x12, 0xf9, 0x2c, 0x8b, 0x8a, 0x13, 0xe1, 0x41,
0xfe, 0x4f, 0x44, 0x82, 0xb1, 0xaf, 0x60, 0x72, 0x25, 0xad, 0xbc, 0xdf, 0xae, 0x7d, 0x27, 0x12,
0xb1, 0xc3, 0xcf, 0x54, 0x19, 0xbd, 0xa8, 0x4a, 0xfc, 0xb2, 0x2a, 0x47, 0x2f, 0xab, 0x92, 0x7c,
0x8a, 0x2a, 0xe3, 0xff, 0x50, 0x25, 0x7f, 0x4a, 0xe0, 0xb4, 0x97, 0xe0, 0x6e, 0x6d, 0x55, 0xa3,
0xc9, 0x3d, 0xef, 0x36, 0x6b, 0xc3, 0x23, 0xda, 0x98, 0x62, 0xe7, 0x1e, 0xe7, 0x95, 0x51, 0x16,
0x17, 0xa9, 0xf7, 0xc7, 0xb7, 0x30, 0xbe, 0x56, 0x58, 0x97, 0x2d, 0xff, 0x8c, 0x0c, 0x34, 0x0b,
0x82, 0x3e, 0x48, 0x23, 0xf0, 0x51, 0x84, 0x24, 0xfb, 0x11, 0x8e, 0x17, 0x4d, 0x67, 0x96, 0xd8,
0xf2, 0x98, 0xea, 0x58, 0xa8, 0xfb, 0x80, 0xb2, 0xed, 0x0c, 0xae, 0x50, 0x5b, 0xd1, 0x97, 0xb0,
0x1f, 0x60, 0xe2, 0xa4, 0x30, 0x7f, 0xc9, 0x9a, 0xee, 0x3d, 0xbd, 0x3c, 0xed, 0xfb, 0x14, 0x68,
0xb1, 0x2b, 0x70, 0x5a, 0x5f, 0xa9, 0x15, 0xea, 0xd6, 0x9d, 0x9a, 0x6c, 0x9c, 0x8a, 0x01, 0xc3,
0x38, 0x1c, 0xff, 0x66, 0x9a, 0x6e, 0xfd, 0x76, 0xcb, 0x3f, 0xa7, 0x64, 0x0f, 0xdd, 0x0d, 0xaf,
0x55, 0x5d, 0x93, 0x24, 0x89, 0xa0, 0x98, 0xbd, 0x81, 0xd4, 0xfd, 0x0e, 0xed, 0xbc, 0x27, 0x5c,
0xf6, 0xd7, 0x46, 0x97, 0xca, 0x29, 0x44, 0x56, 0x4e, 0xc5, 0x9e, 0x70, 0xd9, 0x85, 0x95, 0xc6,
0xd2, 0xd0, 0xa5, 0xd4, 0xd2, 0x3d, 0xe1, 0xce, 0xf1, 0x4e, 0x97, 0x94, 0x03, 0xca, 0xf5, 0xd0,
0x39, 0xe9, 0x7d, 0xb3, 0x94, 0xb4, 0xe8, 0x17, 0xb4, 0xe8, 0x0e, 0xbb, 0x35, 0xe7, 0xed, 0x12,
0x75, 0xa9, 0x74, 0x45, 0x9e, 0x9d, 0x88, 0x3d, 0xe1, 0x1c, 0xfa, 0x5e, 0xad, 0x94, 0x25, 0xaf,
0xc7, 0xc2, 0x03, 0x76, 0x0e, 0xe3, 0xbb, 0xc7, 0xc7, 0x16, 0x2d, 0x19, 0x37, 0x16, 0x01, 0x39,
0x7e, 0xe1, 0xcb, 0x5f, 0x79, 0xde, 0x23, 0x77, 0xb2, 0x45, 0xf8, 0xc3, 0xa9, 0x3f, 0x59, 0x80,
0xfe, 0x46, 0x46, 0xad, 0xe9, 0xb9, 0x39, 0xf7, 0xbb, 0xef, 0x08, 0xb7, 0xde, 0x15, 0x96, 0xdd,
0x1a, 0xf9, 0x6b, 0x4a, 0x05, 0xe4, 0x3a, 0xf2, 0x41, 0x6e, 0x16, 0x68, 0x14, 0xb6, 0xb7, 0x9c,
0xd1, 0x92, 0x03, 0xc6, 0xed, 0x77, 0x67, 0x4a, 0x34, 0x58, 0xf2, 0x33, 0xfa, 0x63, 0x0f, 0xf3,
0x9f, 0xe1, 0x64, 0x60, 0x88, 0x96, 0x15, 0x90, 0xdc, 0x58, 0x5c, 0xb5, 0x3c, 0xfa, 0x5f, 0xd3,
0xf8, 0x82, 0xfc, 0xef, 0x08, 0xa6, 0x03, 0xba, 0x9f, 0xce, 0x3f, 0x64, 0x8b, 0xc1, 0xc1, 0x3b,
0xcc, 0x0a, 0x38, 0x15, 0x68, 0x51, 0x3b, 0x81, 0x3f, 0x36, 0xb5, 0x5a, 0x6e, 0x69, 0x44, 0x53,
0xf1, 0x9c, 0xde, 0xbd, 0xb4, 0xb1, 0x9f, 0x01, 0xba, 0xf5, 0x19, 0x24, 0x02, 0x2b, 0xdc, 0x84,
0x89, 0xf4, 0xc0, 0xed, 0x77, 0xd3, 0xde, 0x4b, 0x53, 0xa1, 0x0d, 0x73, 0xb8, 0xc3, 0xec, 0x3b,
0x78, 0xb5, 0xd8, 0xb6, 0x16, 0x57, 0xfd, 0x88, 0x91, 0xe3, 0x52, 0xf1, 0x8c, 0xcd, 0x7f, 0xd9,
0xdb, 0x9e, 0xce, 0xdf, 0x19, 0xef, 0x89, 0x88, 0x14, 0xdc, 0xe1, 0x41, 0x7f, 0x47, 0xc3, 0xfe,
0xe6, 0x73, 0x98, 0x1d, 0xbc, 0x63, 0xd4, 0xd8, 0xd0, 0x85, 0x28, 0x34, 0x36, 0xb4, 0xe0, 0x1c,
0xc6, 0xf4, 0x2d, 0xb9, 0xed, 0x97, 0xf0, 0x28, 0xbf, 0x80, 0xb1, 0x9f, 0x5c, 0x37, 0xea, 0x0f,
0xb2, 0x0e, 0xdf, 0x18, 0x17, 0xd2, 0xe7, 0xc4, 0x3d, 0x76, 0x23, 0x3f, 0x2e, 0x2e, 0xfe, 0x37,
0x00, 0x00, 0xff, 0xff, 0x07, 0x98, 0x54, 0xa1, 0xb5, 0x06, 0x00, 0x00,
}

82
influxql/query/internal/internal.proto Normal file
View File

@ -0,0 +1,82 @@
syntax = "proto2";
package query;
message Point {
required string Name = 1;
required string Tags = 2;
required int64 Time = 3;
required bool Nil = 4;
repeated Aux Aux = 5;
optional uint32 Aggregated = 6;
optional double FloatValue = 7;
optional int64 IntegerValue = 8;
optional string StringValue = 9;
optional bool BooleanValue = 10;
optional uint64 UnsignedValue = 12;
optional IteratorStats Stats = 11;
optional bytes Trace = 13;
}
message Aux {
required int32 DataType = 1;
optional double FloatValue = 2;
optional int64 IntegerValue = 3;
optional string StringValue = 4;
optional bool BooleanValue = 5;
optional uint64 UnsignedValue = 6;
}
message IteratorOptions {
optional string Expr = 1;
repeated string Aux = 2;
repeated VarRef Fields = 17;
repeated Measurement Sources = 3;
optional Interval Interval = 4;
repeated string Dimensions = 5;
repeated string GroupBy = 19;
optional int32 Fill = 6;
optional double FillValue = 7;
optional string Condition = 8;
optional int64 StartTime = 9;
optional int64 EndTime = 10;
optional string Location = 21;
optional bool Ascending = 11;
optional int64 Limit = 12;
optional int64 Offset = 13;
optional int64 SLimit = 14;
optional int64 SOffset = 15;
optional bool StripName = 22;
optional bool Dedupe = 16;
optional int64 MaxSeriesN = 18;
optional bool Ordered = 20;
}
message Measurements {
repeated Measurement Items = 1;
}
message Measurement {
optional string Database = 1;
optional string RetentionPolicy = 2;
optional string Name = 3;
optional string Regex = 4;
optional bool IsTarget = 5;
optional string SystemIterator = 6;
}
message Interval {
optional int64 Duration = 1;
optional int64 Offset = 2;
}
message IteratorStats {
optional int64 SeriesN = 1;
optional int64 PointN = 2;
}
message VarRef {
required string Val = 1;
optional int32 Type = 2;
}

13529
influxql/query/iterator.gen.go Normal file
View File

File diff suppressed because it is too large Load Diff

1588
influxql/query/iterator.gen.go.tmpl Normal file
View File

File diff suppressed because it is too large Load Diff

1422
influxql/query/iterator.go Normal file
View File

File diff suppressed because it is too large Load Diff

67
influxql/query/iterator_mapper.go Normal file
View File

@ -0,0 +1,67 @@
package query
import (
"fmt"
"math"
"github.com/influxdata/influxql"
)
type IteratorMap interface {
Value(row *Row) interface{}
}
type FieldMap struct {
Index int
Type influxql.DataType
}
func (f FieldMap) Value(row *Row) interface{} {
v := castToType(row.Values[f.Index], f.Type)
if v == NullFloat {
// If the value is a null float, then convert it back to NaN
// so it is treated as a float for eval.
v = math.NaN()
}
return v
}
type TagMap string
func (s TagMap) Value(row *Row) interface{} { return row.Series.Tags.Value(string(s)) }
type NullMap struct{}
func (NullMap) Value(row *Row) interface{} { return nil }
func NewIteratorMapper(cur Cursor, driver IteratorMap, fields []IteratorMap, opt IteratorOptions) Iterator {
if driver != nil {
switch driver := driver.(type) {
case FieldMap:
switch driver.Type {
case influxql.Float:
return newFloatIteratorMapper(cur, driver, fields, opt)
case influxql.Integer:
return newIntegerIteratorMapper(cur, driver, fields, opt)
case influxql.Unsigned:
return newUnsignedIteratorMapper(cur, driver, fields, opt)
case influxql.String, influxql.Tag:
return newStringIteratorMapper(cur, driver, fields, opt)
case influxql.Boolean:
return newBooleanIteratorMapper(cur, driver, fields, opt)
default:
// The driver doesn't appear to to have a valid driver type.
// We should close the cursor and return a blank iterator.
// We close the cursor because we own it and have a responsibility
// to close it once it is passed into this function.
cur.Close()
return &nilFloatIterator{}
}
case TagMap:
return newStringIteratorMapper(cur, driver, fields, opt)
default:
panic(fmt.Sprintf("unable to create iterator mapper with driver expression type: %T", driver))
}
}
return newFloatIteratorMapper(cur, nil, fields, opt)
}

74
influxql/query/iterator_mapper_test.go Normal file
View File

@ -0,0 +1,74 @@
package query_test
import (
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/pkg/deep"
"github.com/influxdata/influxql"
)
func TestIteratorMapper(t *testing.T) {
cur := query.RowCursor([]query.Row{
{
Time: 0,
Series: query.Series{
Name: "cpu",
Tags: ParseTags("host=A"),
},
Values: []interface{}{float64(1), "a"},
},
{
Time: 5,
Series: query.Series{
Name: "cpu",
Tags: ParseTags("host=A"),
},
Values: []interface{}{float64(3), "c"},
},
{
Time: 2,
Series: query.Series{
Name: "cpu",
Tags: ParseTags("host=B"),
},
Values: []interface{}{float64(2), "b"},
},
{
Time: 8,
Series: query.Series{
Name: "cpu",
Tags: ParseTags("host=B"),
},
Values: []interface{}{float64(8), "h"},
},
}, []influxql.VarRef{
{Val: "val1", Type: influxql.Float},
{Val: "val2", Type: influxql.String},
})
opt := query.IteratorOptions{
Ascending: true,
Aux: []influxql.VarRef{
{Val: "val1", Type: influxql.Float},
{Val: "val2", Type: influxql.String},
},
Dimensions: []string{"host"},
}
itr := query.NewIteratorMapper(cur, nil, []query.IteratorMap{
query.FieldMap{Index: 0},
query.FieldMap{Index: 1},
query.TagMap("host"),
}, opt)
if a, err := Iterators([]query.Iterator{itr}).ReadAll(); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if !deep.Equal(a, [][]query.Point{
{&query.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0, Aux: []interface{}{float64(1), "a", "A"}}},
{&query.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 5, Aux: []interface{}{float64(3), "c", "A"}}},
{&query.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 2, Aux: []interface{}{float64(2), "b", "B"}}},
{&query.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 8, Aux: []interface{}{float64(8), "h", "B"}}},
}) {
t.Errorf("unexpected points: %s", spew.Sdump(a))
}
}

1784
influxql/query/iterator_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

31
influxql/query/linear.go Normal file
View File

@ -0,0 +1,31 @@
package query
// linearFloat computes the the slope of the line between the points (previousTime, previousValue) and (nextTime, nextValue)
// and returns the value of the point on the line with time windowTime
// y = mx + b
func linearFloat(windowTime, previousTime, nextTime int64, previousValue, nextValue float64) float64 {
m := (nextValue - previousValue) / float64(nextTime-previousTime) // the slope of the line
x := float64(windowTime - previousTime) // how far into the interval we are
b := previousValue
return m*x + b
}
// linearInteger computes the the slope of the line between the points (previousTime, previousValue) and (nextTime, nextValue)
// and returns the value of the point on the line with time windowTime
// y = mx + b
func linearInteger(windowTime, previousTime, nextTime int64, previousValue, nextValue int64) int64 {
m := float64(nextValue-previousValue) / float64(nextTime-previousTime) // the slope of the line
x := float64(windowTime - previousTime) // how far into the interval we are
b := float64(previousValue)
return int64(m*x + b)
}
// linearInteger computes the the slope of the line between the points (previousTime, previousValue) and (nextTime, nextValue)
// and returns the value of the point on the line with time windowTime
// y = mx + b
func linearUnsigned(windowTime, previousTime, nextTime int64, previousValue, nextValue uint64) uint64 {
m := float64(nextValue-previousValue) / float64(nextTime-previousTime) // the slope of the line
x := float64(windowTime - previousTime) // how far into the interval we are
b := float64(previousValue)
return uint64(m*x + b)
}

246
influxql/query/math.go Normal file
View File

@ -0,0 +1,246 @@
package query
import (
"fmt"
"math"
"github.com/influxdata/influxql"
)
func isMathFunction(call *influxql.Call) bool {
switch call.Name {
case "abs", "sin", "cos", "tan", "asin", "acos", "atan", "atan2", "exp", "log", "ln", "log2", "log10", "sqrt", "pow", "floor", "ceil", "round":
return true
}
return false
}
type MathTypeMapper struct{}
func (MathTypeMapper) MapType(measurement *influxql.Measurement, field string) influxql.DataType {
return influxql.Unknown
}
func (MathTypeMapper) CallType(name string, args []influxql.DataType) (influxql.DataType, error) {
switch name {
case "sin", "cos", "tan", "atan", "exp", "log", "ln", "log2", "log10", "sqrt":
var arg0 influxql.DataType
if len(args) > 0 {
arg0 = args[0]
}
switch arg0 {
case influxql.Float, influxql.Integer, influxql.Unsigned, influxql.Unknown:
return influxql.Float, nil
default:
return influxql.Unknown, fmt.Errorf("invalid argument type for the first argument in %s(): %s", name, arg0)
}
case "asin", "acos":
var arg0 influxql.DataType
if len(args) > 0 {
arg0 = args[0]
}
switch arg0 {
case influxql.Float, influxql.Unknown:
return influxql.Float, nil
default:
return influxql.Unknown, fmt.Errorf("invalid argument type for the first argument in %s(): %s", name, arg0)
}
case "atan2", "pow":
var arg0, arg1 influxql.DataType
if len(args) > 0 {
arg0 = args[0]
}
if len(args) > 1 {
arg1 = args[1]
}
switch arg0 {
case influxql.Float, influxql.Integer, influxql.Unsigned, influxql.Unknown:
// Pass through to verify the second argument.
default:
return influxql.Unknown, fmt.Errorf("invalid argument type for the first argument in %s(): %s", name, arg0)
}
switch arg1 {
case influxql.Float, influxql.Integer, influxql.Unsigned, influxql.Unknown:
return influxql.Float, nil
default:
return influxql.Unknown, fmt.Errorf("invalid argument type for the second argument in %s(): %s", name, arg1)
}
case "abs", "floor", "ceil", "round":
var arg0 influxql.DataType
if len(args) > 0 {
arg0 = args[0]
}
switch arg0 {
case influxql.Float, influxql.Integer, influxql.Unsigned, influxql.Unknown:
return args[0], nil
default:
return influxql.Unknown, fmt.Errorf("invalid argument type for the first argument in %s(): %s", name, arg0)
}
}
return influxql.Unknown, nil
}
type MathValuer struct{}
var _ influxql.CallValuer = MathValuer{}
func (MathValuer) Value(key string) (interface{}, bool) {
return nil, false
}
func (v MathValuer) Call(name string, args []interface{}) (interface{}, bool) {
if len(args) == 1 {
arg0 := args[0]
switch name {
case "abs":
switch arg0 := arg0.(type) {
case float64:
return math.Abs(arg0), true
case int64:
sign := arg0 >> 63
return (arg0 ^ sign) - sign, true
case uint64:
return arg0, true
default:
return nil, true
}
case "sin":
if arg0, ok := asFloat(arg0); ok {
return math.Sin(arg0), true
}
return nil, true
case "cos":
if arg0, ok := asFloat(arg0); ok {
return math.Cos(arg0), true
}
return nil, true
case "tan":
if arg0, ok := asFloat(arg0); ok {
return math.Tan(arg0), true
}
return nil, true
case "floor":
switch arg0 := arg0.(type) {
case float64:
return math.Floor(arg0), true
case int64, uint64:
return arg0, true
default:
return nil, true
}
case "ceil":
switch arg0 := arg0.(type) {
case float64:
return math.Ceil(arg0), true
case int64, uint64:
return arg0, true
default:
return nil, true
}
case "round":
switch arg0 := arg0.(type) {
case float64:
return round(arg0), true
case int64, uint64:
return arg0, true
default:
return nil, true
}
case "asin":
if arg0, ok := asFloat(arg0); ok {
return math.Asin(arg0), true
}
return nil, true
case "acos":
if arg0, ok := asFloat(arg0); ok {
return math.Acos(arg0), true
}
return nil, true
case "atan":
if arg0, ok := asFloat(arg0); ok {
return math.Atan(arg0), true
}
return nil, true
case "exp":
if arg0, ok := asFloat(arg0); ok {
return math.Exp(arg0), true
}
return nil, true
case "ln":
if arg0, ok := asFloat(arg0); ok {
return math.Log(arg0), true
}
return nil, true
case "log2":
if arg0, ok := asFloat(arg0); ok {
return math.Log2(arg0), true
}
return nil, true
case "log10":
if arg0, ok := asFloat(arg0); ok {
return math.Log10(arg0), true
}
return nil, true
case "sqrt":
if arg0, ok := asFloat(arg0); ok {
return math.Sqrt(arg0), true
}
return nil, true
}
} else if len(args) == 2 {
arg0, arg1 := args[0], args[1]
switch name {
case "atan2":
if arg0, arg1, ok := asFloats(arg0, arg1); ok {
return math.Atan2(arg0, arg1), true
}
return nil, true
case "log":
if arg0, arg1, ok := asFloats(arg0, arg1); ok {
return math.Log(arg0) / math.Log(arg1), true
}
return nil, true
case "pow":
if arg0, arg1, ok := asFloats(arg0, arg1); ok {
return math.Pow(arg0, arg1), true
}
return nil, true
}
}
return nil, false
}
func asFloat(x interface{}) (float64, bool) {
switch arg0 := x.(type) {
case float64:
return arg0, true
case int64:
return float64(arg0), true
case uint64:
return float64(arg0), true
default:
return 0, false
}
}
func asFloats(x, y interface{}) (float64, float64, bool) {
arg0, ok := asFloat(x)
if !ok {
return 0, 0, false
}
arg1, ok := asFloat(y)
if !ok {
return 0, 0, false
}
return arg0, arg1, true
}
func round(x float64) float64 {
t := math.Trunc(x)
if math.Abs(x-t) >= 0.5 {
return t + math.Copysign(1, x)
}
return t
}

212
influxql/query/math_test.go Normal file
View File

@ -0,0 +1,212 @@
package query_test
import (
"math"
"testing"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxql"
)
func TestMath_TypeMapper(t *testing.T) {
for _, tt := range []struct {
s string
typ influxql.DataType
err bool
}{
{s: `abs(f::float)`, typ: influxql.Float},
{s: `abs(i::integer)`, typ: influxql.Integer},
{s: `abs(u::unsigned)`, typ: influxql.Unsigned},
{s: `abs(s::string)`, err: true},
{s: `abs(b::boolean)`, err: true},
{s: `sin(f::float)`, typ: influxql.Float},
{s: `sin(i::integer)`, typ: influxql.Float},
{s: `sin(u::unsigned)`, typ: influxql.Float},
{s: `sin(s::string)`, err: true},
{s: `sin(b::boolean)`, err: true},
{s: `cos(f::float)`, typ: influxql.Float},
{s: `cos(i::integer)`, typ: influxql.Float},
{s: `cos(u::unsigned)`, typ: influxql.Float},
{s: `cos(s::string)`, err: true},
{s: `cos(b::boolean)`, err: true},
{s: `tan(f::float)`, typ: influxql.Float},
{s: `tan(i::integer)`, typ: influxql.Float},
{s: `tan(u::unsigned)`, typ: influxql.Float},
{s: `tan(s::string)`, err: true},
{s: `tan(b::boolean)`, err: true},
{s: `asin(f::float)`, typ: influxql.Float},
{s: `asin(i::integer)`, err: true},
{s: `asin(u::unsigned)`, err: true},
{s: `asin(s::string)`, err: true},
{s: `asin(b::boolean)`, err: true},
{s: `acos(f::float)`, typ: influxql.Float},
{s: `acos(i::integer)`, err: true},
{s: `acos(u::unsigned)`, err: true},
{s: `acos(s::string)`, err: true},
{s: `acos(b::boolean)`, err: true},
{s: `atan(f::float)`, typ: influxql.Float},
{s: `atan(i::integer)`, typ: influxql.Float},
{s: `atan(u::unsigned)`, typ: influxql.Float},
{s: `atan(s::string)`, err: true},
{s: `atan(b::boolean)`, err: true},
{s: `atan2(y::float, x::float)`, typ: influxql.Float},
{s: `atan2(y::integer, x::float)`, typ: influxql.Float},
{s: `atan2(y::unsigned, x::float)`, typ: influxql.Float},
{s: `atan2(y::string, x::float)`, err: true},
{s: `atan2(y::boolean, x::float)`, err: true},
{s: `atan2(y::float, x::float)`, typ: influxql.Float},
{s: `atan2(y::float, x::integer)`, typ: influxql.Float},
{s: `atan2(y::float, x::unsigned)`, typ: influxql.Float},
{s: `atan2(y::float, x::string)`, err: true},
{s: `atan2(y::float, x::boolean)`, err: true},
{s: `exp(f::float)`, typ: influxql.Float},
{s: `exp(i::integer)`, typ: influxql.Float},
{s: `exp(u::unsigned)`, typ: influxql.Float},
{s: `exp(s::string)`, err: true},
{s: `exp(b::boolean)`, err: true},
{s: `log(f::float)`, typ: influxql.Float},
{s: `log(i::integer)`, typ: influxql.Float},
{s: `log(u::unsigned)`, typ: influxql.Float},
{s: `log(s::string)`, err: true},
{s: `log(b::boolean)`, err: true},
{s: `ln(f::float)`, typ: influxql.Float},
{s: `ln(i::integer)`, typ: influxql.Float},
{s: `ln(u::unsigned)`, typ: influxql.Float},
{s: `ln(s::string)`, err: true},
{s: `ln(b::boolean)`, err: true},
{s: `log2(f::float)`, typ: influxql.Float},
{s: `log2(i::integer)`, typ: influxql.Float},
{s: `log2(u::unsigned)`, typ: influxql.Float},
{s: `log2(s::string)`, err: true},
{s: `log2(b::boolean)`, err: true},
{s: `log10(f::float)`, typ: influxql.Float},
{s: `log10(i::integer)`, typ: influxql.Float},
{s: `log10(u::unsigned)`, typ: influxql.Float},
{s: `log10(s::string)`, err: true},
{s: `log10(b::boolean)`, err: true},
{s: `sqrt(f::float)`, typ: influxql.Float},
{s: `sqrt(i::integer)`, typ: influxql.Float},
{s: `sqrt(u::unsigned)`, typ: influxql.Float},
{s: `sqrt(s::string)`, err: true},
{s: `sqrt(b::boolean)`, err: true},
{s: `pow(y::float, x::float)`, typ: influxql.Float},
{s: `pow(y::integer, x::float)`, typ: influxql.Float},
{s: `pow(y::unsigned, x::float)`, typ: influxql.Float},
{s: `pow(y::string, x::string)`, err: true},
{s: `pow(y::boolean, x::boolean)`, err: true},
{s: `pow(y::float, x::float)`, typ: influxql.Float},
{s: `pow(y::float, x::integer)`, typ: influxql.Float},
{s: `pow(y::float, x::unsigned)`, typ: influxql.Float},
{s: `pow(y::float, x::string)`, err: true},
{s: `pow(y::float, x::boolean)`, err: true},
{s: `floor(f::float)`, typ: influxql.Float},
{s: `floor(i::integer)`, typ: influxql.Integer},
{s: `floor(u::unsigned)`, typ: influxql.Unsigned},
{s: `floor(s::string)`, err: true},
{s: `floor(b::boolean)`, err: true},
{s: `ceil(f::float)`, typ: influxql.Float},
{s: `ceil(i::integer)`, typ: influxql.Integer},
{s: `ceil(u::unsigned)`, typ: influxql.Unsigned},
{s: `ceil(s::string)`, err: true},
{s: `ceil(b::boolean)`, err: true},
{s: `round(f::float)`, typ: influxql.Float},
{s: `round(i::integer)`, typ: influxql.Integer},
{s: `round(u::unsigned)`, typ: influxql.Unsigned},
{s: `round(s::string)`, err: true},
{s: `round(b::boolean)`, err: true},
} {
t.Run(tt.s, func(t *testing.T) {
expr := MustParseExpr(tt.s)
typmap := influxql.TypeValuerEval{
TypeMapper: query.MathTypeMapper{},
}
if got, err := typmap.EvalType(expr); err != nil {
if !tt.err {
t.Errorf("unexpected error: %s", err)
}
} else if tt.err {
t.Error("expected error")
} else if want := tt.typ; got != want {
t.Errorf("unexpected type:\n\t-: \"%s\"\n\t+: \"%s\"", want, got)
}
})
}
}
func TestMathValuer_Call(t *testing.T) {
type values map[string]interface{}
for _, tt := range []struct {
s string
values values
exp interface{}
}{
{s: `abs(f)`, values: values{"f": float64(2)}, exp: float64(2)},
{s: `abs(f)`, values: values{"f": float64(-2)}, exp: float64(2)},
{s: `abs(i)`, values: values{"i": int64(2)}, exp: int64(2)},
{s: `abs(i)`, values: values{"i": int64(-2)}, exp: int64(2)},
{s: `abs(u)`, values: values{"u": uint64(2)}, exp: uint64(2)},
{s: `sin(f)`, values: values{"f": math.Pi / 2}, exp: math.Sin(math.Pi / 2)},
{s: `sin(i)`, values: values{"i": int64(2)}, exp: math.Sin(2)},
{s: `sin(u)`, values: values{"u": uint64(2)}, exp: math.Sin(2)},
{s: `asin(f)`, values: values{"f": float64(0.5)}, exp: math.Asin(0.5)},
{s: `cos(f)`, values: values{"f": math.Pi / 2}, exp: math.Cos(math.Pi / 2)},
{s: `cos(i)`, values: values{"i": int64(2)}, exp: math.Cos(2)},
{s: `cos(u)`, values: values{"u": uint64(2)}, exp: math.Cos(2)},
{s: `acos(f)`, values: values{"f": float64(0.5)}, exp: math.Acos(0.5)},
{s: `tan(f)`, values: values{"f": math.Pi / 2}, exp: math.Tan(math.Pi / 2)},
{s: `tan(i)`, values: values{"i": int64(2)}, exp: math.Tan(2)},
{s: `tan(u)`, values: values{"u": uint64(2)}, exp: math.Tan(2)},
{s: `atan(f)`, values: values{"f": float64(2)}, exp: math.Atan(2)},
{s: `atan(i)`, values: values{"i": int64(2)}, exp: math.Atan(2)},
{s: `atan(u)`, values: values{"u": uint64(2)}, exp: math.Atan(2)},
{s: `atan2(y, x)`, values: values{"y": float64(2), "x": float64(3)}, exp: math.Atan2(2, 3)},
{s: `atan2(y, x)`, values: values{"y": int64(2), "x": int64(3)}, exp: math.Atan2(2, 3)},
{s: `atan2(y, x)`, values: values{"y": uint64(2), "x": uint64(3)}, exp: math.Atan2(2, 3)},
{s: `floor(f)`, values: values{"f": float64(2.5)}, exp: float64(2)},
{s: `floor(i)`, values: values{"i": int64(2)}, exp: int64(2)},
{s: `floor(u)`, values: values{"u": uint64(2)}, exp: uint64(2)},
{s: `ceil(f)`, values: values{"f": float64(2.5)}, exp: float64(3)},
{s: `ceil(i)`, values: values{"i": int64(2)}, exp: int64(2)},
{s: `ceil(u)`, values: values{"u": uint64(2)}, exp: uint64(2)},
{s: `round(f)`, values: values{"f": float64(2.4)}, exp: float64(2)},
{s: `round(f)`, values: values{"f": float64(2.6)}, exp: float64(3)},
{s: `round(i)`, values: values{"i": int64(2)}, exp: int64(2)},
{s: `round(u)`, values: values{"u": uint64(2)}, exp: uint64(2)},
{s: `exp(f)`, values: values{"f": float64(3)}, exp: math.Exp(3)},
{s: `exp(i)`, values: values{"i": int64(3)}, exp: math.Exp(3)},
{s: `exp(u)`, values: values{"u": uint64(3)}, exp: math.Exp(3)},
{s: `log(f, 8)`, values: values{"f": float64(3)}, exp: math.Log(3) / math.Log(8)},
{s: `log(i, 8)`, values: values{"i": int64(3)}, exp: math.Log(3) / math.Log(8)},
{s: `log(u, 8)`, values: values{"u": uint64(3)}, exp: math.Log(3) / math.Log(8)},
{s: `ln(f)`, values: values{"f": float64(3)}, exp: math.Log(3)},
{s: `ln(i)`, values: values{"i": int64(3)}, exp: math.Log(3)},
{s: `ln(u)`, values: values{"u": uint64(3)}, exp: math.Log(3)},
{s: `log2(f)`, values: values{"f": float64(3)}, exp: math.Log2(3)},
{s: `log2(i)`, values: values{"i": int64(3)}, exp: math.Log2(3)},
{s: `log2(u)`, values: values{"u": uint64(3)}, exp: math.Log2(3)},
{s: `log10(f)`, values: values{"f": float64(3)}, exp: math.Log10(3)},
{s: `log10(i)`, values: values{"i": int64(3)}, exp: math.Log10(3)},
{s: `log10(u)`, values: values{"u": uint64(3)}, exp: math.Log10(3)},
{s: `sqrt(f)`, values: values{"f": float64(3)}, exp: math.Sqrt(3)},
{s: `sqrt(i)`, values: values{"i": int64(3)}, exp: math.Sqrt(3)},
{s: `sqrt(u)`, values: values{"u": uint64(3)}, exp: math.Sqrt(3)},
{s: `pow(f, 2)`, values: values{"f": float64(4)}, exp: math.Pow(4, 2)},
{s: `pow(i, 2)`, values: values{"i": int64(4)}, exp: math.Pow(4, 2)},
{s: `pow(u, 2)`, values: values{"u": uint64(4)}, exp: math.Pow(4, 2)},
} {
t.Run(tt.s, func(t *testing.T) {
expr := MustParseExpr(tt.s)
valuer := influxql.ValuerEval{
Valuer: influxql.MultiValuer(
influxql.MapValuer(tt.values),
query.MathValuer{},
),
}
if got, want := valuer.Eval(expr), tt.exp; got != want {
t.Errorf("unexpected value: %v != %v", want, got)
}
})
}
}

48
influxql/query/monitor.go Normal file
View File

@ -0,0 +1,48 @@
package query
import (
"context"
"time"
)
// MonitorFunc is a function that will be called to check if a query
// is currently healthy. If the query needs to be interrupted for some reason,
// the error should be returned by this function.
type MonitorFunc func(<-chan struct{}) error
// Monitor monitors the status of a query and returns whether the query should
// be aborted with an error.
type Monitor interface {
// Monitor starts a new goroutine that will monitor a query. The function
// will be passed in a channel to signal when the query has been finished
// normally. If the function returns with an error and the query is still
// running, the query will be terminated.
Monitor(fn MonitorFunc)
}
// MonitorFromContext returns a Monitor embedded within the Context
// if one exists.
func MonitorFromContext(ctx context.Context) Monitor {
v, _ := ctx.Value(monitorContextKey{}).(Monitor)
return v
}
// PointLimitMonitor is a query monitor that exits when the number of points
// emitted exceeds a threshold.
func PointLimitMonitor(cur Cursor, interval time.Duration, limit int) MonitorFunc {
return func(closing <-chan struct{}) error {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
stats := cur.Stats()
if stats.PointN >= limit {
return ErrMaxSelectPointsLimitExceeded(stats.PointN, limit)
}
case <-closing:
return nil
}
}
}
}

61
influxql/query/monitor_test.go Normal file
View File

@ -0,0 +1,61 @@
package query_test
import (
"context"
"testing"
"time"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxql"
)
func TestPointLimitMonitor(t *testing.T) {
t.Parallel()
stmt := MustParseSelectStatement(`SELECT mean(value) FROM cpu`)
// Create a new task manager so we can use the query task as a monitor.
taskManager := query.NewTaskManager()
ctx, detach, err := taskManager.AttachQuery(&influxql.Query{
Statements: []influxql.Statement{stmt},
}, query.ExecutionOptions{}, nil)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
defer detach()
shardMapper := ShardMapper{
MapShardsFn: func(sources influxql.Sources, t influxql.TimeRange) query.ShardGroup {
return &ShardGroup{
CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) {
return &FloatIterator{
Points: []query.FloatPoint{
{Name: "cpu", Value: 35},
},
Context: ctx,
Delay: 2 * time.Second,
stats: query.IteratorStats{
PointN: 10,
},
}, nil
},
Fields: map[string]influxql.DataType{
"value": influxql.Float,
},
}
},
}
cur, err := query.Select(ctx, stmt, &shardMapper, query.SelectOptions{
MaxPointN: 1,
})
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if err := query.DrainCursor(cur); err == nil {
t.Fatalf("expected an error")
} else if got, want := err.Error(), "max-select-point limit exceeed: (10/1)"; got != want {
t.Fatalf("unexpected error: got=%v want=%v", got, want)
}
}

239
influxql/query/neldermead/neldermead.go Normal file
View File

@ -0,0 +1,239 @@
// Package neldermead is an implementation of the Nelder-Mead optimization method.
// Based on work by Michael F. Hutt: http://www.mikehutt.com/neldermead.html
package neldermead
import "math"
const (
defaultMaxIterations = 1000
// reflection coefficient
defaultAlpha = 1.0
// contraction coefficient
defaultBeta = 0.5
// expansion coefficient
defaultGamma = 2.0
)
// Optimizer represents the parameters to the Nelder-Mead simplex method.
type Optimizer struct {
// Maximum number of iterations.
MaxIterations int
// Reflection coefficient.
Alpha,
// Contraction coefficient.
Beta,
// Expansion coefficient.
Gamma float64
}
// New returns a new instance of Optimizer with all values set to the defaults.
func New() *Optimizer {
return &Optimizer{
MaxIterations: defaultMaxIterations,
Alpha: defaultAlpha,
Beta: defaultBeta,
Gamma: defaultGamma,
}
}
// Optimize applies the Nelder-Mead simplex method with the Optimizer's settings.
func (o *Optimizer) Optimize(
objfunc func([]float64) float64,
start []float64,
epsilon,
scale float64,
) (float64, []float64) {
n := len(start)
//holds vertices of simplex
v := make([][]float64, n+1)
for i := range v {
v[i] = make([]float64, n)
}
//value of function at each vertex
f := make([]float64, n+1)
//reflection - coordinates
vr := make([]float64, n)
//expansion - coordinates
ve := make([]float64, n)
//contraction - coordinates
vc := make([]float64, n)
//centroid - coordinates
vm := make([]float64, n)
// create the initial simplex
// assume one of the vertices is 0,0
pn := scale * (math.Sqrt(float64(n+1)) - 1 + float64(n)) / (float64(n) * math.Sqrt(2))
qn := scale * (math.Sqrt(float64(n+1)) - 1) / (float64(n) * math.Sqrt(2))
for i := 0; i < n; i++ {
v[0][i] = start[i]
}
for i := 1; i <= n; i++ {
for j := 0; j < n; j++ {
if i-1 == j {
v[i][j] = pn + start[j]
} else {
v[i][j] = qn + start[j]
}
}
}
// find the initial function values
for j := 0; j <= n; j++ {
f[j] = objfunc(v[j])
}
// begin the main loop of the minimization
for itr := 1; itr <= o.MaxIterations; itr++ {
// find the indexes of the largest and smallest values
vg := 0
vs := 0
for i := 0; i <= n; i++ {
if f[i] > f[vg] {
vg = i
}
if f[i] < f[vs] {
vs = i
}
}
// find the index of the second largest value
vh := vs
for i := 0; i <= n; i++ {
if f[i] > f[vh] && f[i] < f[vg] {
vh = i
}
}
// calculate the centroid
for i := 0; i <= n-1; i++ {
cent := 0.0
for m := 0; m <= n; m++ {
if m != vg {
cent += v[m][i]
}
}
vm[i] = cent / float64(n)
}
// reflect vg to new vertex vr
for i := 0; i <= n-1; i++ {
vr[i] = vm[i] + o.Alpha*(vm[i]-v[vg][i])
}
// value of function at reflection point
fr := objfunc(vr)
if fr < f[vh] && fr >= f[vs] {
for i := 0; i <= n-1; i++ {
v[vg][i] = vr[i]
}
f[vg] = fr
}
// investigate a step further in this direction
if fr < f[vs] {
for i := 0; i <= n-1; i++ {
ve[i] = vm[i] + o.Gamma*(vr[i]-vm[i])
}
// value of function at expansion point
fe := objfunc(ve)
// by making fe < fr as opposed to fe < f[vs],
// Rosenbrocks function takes 63 iterations as opposed
// to 64 when using double variables.
if fe < fr {
for i := 0; i <= n-1; i++ {
v[vg][i] = ve[i]
}
f[vg] = fe
} else {
for i := 0; i <= n-1; i++ {
v[vg][i] = vr[i]
}
f[vg] = fr
}
}
// check to see if a contraction is necessary
if fr >= f[vh] {
if fr < f[vg] && fr >= f[vh] {
// perform outside contraction
for i := 0; i <= n-1; i++ {
vc[i] = vm[i] + o.Beta*(vr[i]-vm[i])
}
} else {
// perform inside contraction
for i := 0; i <= n-1; i++ {
vc[i] = vm[i] - o.Beta*(vm[i]-v[vg][i])
}
}
// value of function at contraction point
fc := objfunc(vc)
if fc < f[vg] {
for i := 0; i <= n-1; i++ {
v[vg][i] = vc[i]
}
f[vg] = fc
} else {
// at this point the contraction is not successful,
// we must halve the distance from vs to all the
// vertices of the simplex and then continue.
for row := 0; row <= n; row++ {
if row != vs {
for i := 0; i <= n-1; i++ {
v[row][i] = v[vs][i] + (v[row][i]-v[vs][i])/2.0
}
}
}
f[vg] = objfunc(v[vg])
f[vh] = objfunc(v[vh])
}
}
// test for convergence
fsum := 0.0
for i := 0; i <= n; i++ {
fsum += f[i]
}
favg := fsum / float64(n+1)
s := 0.0
for i := 0; i <= n; i++ {
s += math.Pow((f[i]-favg), 2.0) / float64(n)
}
s = math.Sqrt(s)
if s < epsilon {
break
}
}
// find the index of the smallest value
vs := 0
for i := 0; i <= n; i++ {
if f[i] < f[vs] {
vs = i
}
}
parameters := make([]float64, n)
for i := 0; i < n; i++ {
parameters[i] = v[vs][i]
}
min := objfunc(v[vs])
return min, parameters
}

64
influxql/query/neldermead/neldermead_test.go Normal file
View File

@ -0,0 +1,64 @@
package neldermead_test
import (
"math"
"testing"
"github.com/influxdata/influxdb/v2/influxql/query/neldermead"
)
func round(num float64, precision float64) float64 {
rnum := num * math.Pow(10, precision)
var tnum float64
if rnum < 0 {
tnum = math.Floor(rnum - 0.5)
} else {
tnum = math.Floor(rnum + 0.5)
}
rnum = tnum / math.Pow(10, precision)
return rnum
}
func almostEqual(a, b, e float64) bool {
return math.Abs(a-b) < e
}
func Test_Optimize(t *testing.T) {
constraints := func(x []float64) {
for i := range x {
x[i] = round(x[i], 5)
}
}
// 100*(b-a^2)^2 + (1-a)^2
//
// Obvious global minimum at (a,b) = (1,1)
//
// Useful visualization:
// https://www.wolframalpha.com/input/?i=minimize(100*(b-a%5E2)%5E2+%2B+(1-a)%5E2)
f := func(x []float64) float64 {
constraints(x)
// a = x[0]
// b = x[1]
return 100*(x[1]-x[0]*x[0])*(x[1]-x[0]*x[0]) + (1.0-x[0])*(1.0-x[0])
}
start := []float64{-1.2, 1.0}
opt := neldermead.New()
epsilon := 1e-5
min, parameters := opt.Optimize(f, start, epsilon, 1)
if !almostEqual(min, 0, epsilon) {
t.Errorf("unexpected min: got %f exp 0", min)
}
if !almostEqual(parameters[0], 1, 1e-2) {
t.Errorf("unexpected parameters[0]: got %f exp 1", parameters[0])
}
if !almostEqual(parameters[1], 1, 1e-2) {
t.Errorf("unexpected parameters[1]: got %f exp 1", parameters[1])
}
}

1139
influxql/query/point.gen.go Normal file
View File

File diff suppressed because it is too large Load Diff

250
influxql/query/point.gen.go.tmpl Normal file
View File

@ -0,0 +1,250 @@
package query
import (
"context"
"encoding/binary"
"io"
"github.com/gogo/protobuf/proto"
internal "github.com/influxdata/influxdb/v2/influxql/query/internal"
)
{{range .}}
// {{.Name}}Point represents a point with a {{.Type}} value.
// DO NOT ADD ADDITIONAL FIELDS TO THIS STRUCT.
// See TestPoint_Fields in influxql/point_test.go for more details.
type {{.Name}}Point struct {
Name string
Tags Tags
Time int64
Value {{.Type}}
Aux []interface{}
// Total number of points that were combined into this point from an aggregate.
// If this is zero, the point is not the result of an aggregate function.
Aggregated uint32
Nil bool
}
func (v *{{.Name}}Point) name() string { return v.Name }
func (v *{{.Name}}Point) tags() Tags { return v.Tags }
func (v *{{.Name}}Point) time() int64 { return v.Time }
func (v *{{.Name}}Point) nil() bool { return v.Nil }
func (v *{{.Name}}Point) value() interface{} {
if v.Nil {
return nil
}
return v.Value
}
func (v *{{.Name}}Point) aux() []interface{} { return v.Aux }
// Clone returns a copy of v.
func (v *{{.Name}}Point) Clone() *{{.Name}}Point {
if v == nil {
return nil
}
other := *v
if v.Aux != nil {
other.Aux = make([]interface{}, len(v.Aux))
copy(other.Aux, v.Aux)
}
return &other
}
// CopyTo makes a deep copy into the point.
func (v *{{.Name}}Point) CopyTo(other *{{.Name}}Point) {
other.Name, other.Tags = v.Name, v.Tags
other.Time = v.Time
other.Value, other.Nil = v.Value, v.Nil
if v.Aux != nil {
if len(other.Aux) != len(v.Aux) {
other.Aux = make([]interface{}, len(v.Aux))
}
copy(other.Aux, v.Aux)
}
}
func encode{{.Name}}Point(p *{{.Name}}Point) *internal.Point {
return &internal.Point{
Name: proto.String(p.Name),
Tags: proto.String(p.Tags.ID()),
Time: proto.Int64(p.Time),
Nil: proto.Bool(p.Nil),
Aux: encodeAux(p.Aux),
Aggregated: proto.Uint32(p.Aggregated),
{{if eq .Name "Float"}}
FloatValue: proto.Float64(p.Value),
{{else if eq .Name "Integer"}}
IntegerValue: proto.Int64(p.Value),
{{else if eq .Name "String"}}
StringValue: proto.String(p.Value),
{{else if eq .Name "Boolean"}}
BooleanValue: proto.Bool(p.Value),
{{end}}
}
}
func decode{{.Name}}Point(pb *internal.Point) *{{.Name}}Point {
return &{{.Name}}Point{
Name: pb.GetName(),
Tags: newTagsID(pb.GetTags()),
Time: pb.GetTime(),
Nil: pb.GetNil(),
Aux: decodeAux(pb.Aux),
Aggregated: pb.GetAggregated(),
Value: pb.Get{{.Name}}Value(),
}
}
// {{.name}}Points represents a slice of points sortable by value.
type {{.name}}Points []{{.Name}}Point
func (a {{.name}}Points) Len() int { return len(a) }
func (a {{.name}}Points) Less(i, j int) bool {
if a[i].Time != a[j].Time {
return a[i].Time < a[j].Time
}
return {{if ne .Name "Boolean"}}a[i].Value < a[j].Value{{else}}!a[i].Value{{end}}
}
func (a {{.name}}Points) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
// {{.name}}PointsByValue represents a slice of points sortable by value.
type {{.name}}PointsByValue []{{.Name}}Point
func (a {{.name}}PointsByValue) Len() int { return len(a) }
{{if eq .Name "Boolean"}}
func (a {{.name}}PointsByValue) Less(i, j int) bool { return !a[i].Value }
{{else}}
func (a {{.name}}PointsByValue) Less(i, j int) bool { return a[i].Value < a[j].Value }
{{end}}
func (a {{.name}}PointsByValue) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
// {{.name}}PointsByTime represents a slice of points sortable by value.
type {{.name}}PointsByTime []{{.Name}}Point
func (a {{.name}}PointsByTime) Len() int { return len(a) }
func (a {{.name}}PointsByTime) Less(i, j int) bool { return a[i].Time < a[j].Time }
func (a {{.name}}PointsByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
// {{.name}}PointByFunc represents a slice of points sortable by a function.
type {{.name}}PointsByFunc struct {
points []{{.Name}}Point
cmp func(a, b *{{.Name}}Point) bool
}
func (a *{{.name}}PointsByFunc) Len() int { return len(a.points) }
func (a *{{.name}}PointsByFunc) Less(i, j int) bool { return a.cmp(&a.points[i], &a.points[j]) }
func (a *{{.name}}PointsByFunc) Swap(i, j int) { a.points[i], a.points[j] = a.points[j], a.points[i] }
func (a *{{.name}}PointsByFunc) Push(x interface{}) {
a.points = append(a.points, x.({{.Name}}Point))
}
func (a *{{.name}}PointsByFunc) Pop() interface{} {
p := a.points[len(a.points)-1]
a.points = a.points[:len(a.points)-1]
return p
}
func {{.name}}PointsSortBy(points []{{.Name}}Point, cmp func(a, b *{{.Name}}Point) bool) *{{.name}}PointsByFunc {
return &{{.name}}PointsByFunc{
points: points,
cmp: cmp,
}
}
// {{.Name}}PointEncoder encodes {{.Name}}Point points to a writer.
type {{.Name}}PointEncoder struct {
w io.Writer
}
// New{{.Name}}PointEncoder returns a new instance of {{.Name}}PointEncoder that writes to w.
func New{{.Name}}PointEncoder(w io.Writer) *{{.Name}}PointEncoder {
return &{{.Name}}PointEncoder{w: w}
}
// Encode{{.Name}}Point marshals and writes p to the underlying writer.
func (enc *{{.Name}}PointEncoder) Encode{{.Name}}Point(p *{{.Name}}Point) error {
// Marshal to bytes.
buf, err := proto.Marshal(encode{{.Name}}Point(p))
if err != nil {
return err
}
// Write the length.
if err := binary.Write(enc.w, binary.BigEndian, uint32(len(buf))); err != nil {
return err
}
// Write the encoded point.
if _, err := enc.w.Write(buf); err != nil {
return err
}
return nil
}
// {{.Name}}PointDecoder decodes {{.Name}}Point points from a reader.
type {{.Name}}PointDecoder struct {
r io.Reader
stats IteratorStats
ctx context.Context
}
// New{{.Name}}PointDecoder returns a new instance of {{.Name}}PointDecoder that reads from r.
func New{{.Name}}PointDecoder(ctx context.Context, r io.Reader) *{{.Name}}PointDecoder {
return &{{.Name}}PointDecoder{r: r, ctx: ctx}
}
// Stats returns iterator stats embedded within the stream.
func (dec *{{.Name}}PointDecoder) Stats() IteratorStats { return dec.stats }
// Decode{{.Name}}Point reads from the underlying reader and unmarshals into p.
func (dec *{{.Name}}PointDecoder) Decode{{.Name}}Point(p *{{.Name}}Point) error {
for {
// Read length.
var sz uint32
if err := binary.Read(dec.r, binary.BigEndian, &sz); err != nil {
return err
}
// Read point data.
buf := make([]byte, sz)
if _, err := io.ReadFull(dec.r, buf); err != nil {
return err
}
// Unmarshal into point.
var pb internal.Point
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
// If the point contains stats then read stats and retry.
if pb.Stats != nil {
dec.stats = decodeIteratorStats(pb.Stats)
continue
}
if len(pb.Trace) > 0 {
var err error
err = decodeIteratorTrace(dec.ctx, pb.Trace)
if err != nil {
return err
}
continue
}
// Decode into point object.
*p = *decode{{.Name}}Point(&pb)
return nil
}
}
{{end}}

382
influxql/query/point.go Normal file
View File

@ -0,0 +1,382 @@
package query
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"math"
"sort"
"github.com/gogo/protobuf/proto"
internal "github.com/influxdata/influxdb/v2/influxql/query/internal"
"github.com/influxdata/influxql"
)
// ZeroTime is the Unix nanosecond timestamp for no time.
// This time is not used by the query engine or the storage engine as a valid time.
const ZeroTime = int64(math.MinInt64)
// Point represents a value in a series that occurred at a given time.
type Point interface {
// Name and tags uniquely identify the series the value belongs to.
name() string
tags() Tags
// The time that the value occurred at.
time() int64
// The value at the given time.
value() interface{}
// Auxillary values passed along with the value.
aux() []interface{}
}
// Points represents a list of points.
type Points []Point
// Clone returns a deep copy of a.
func (a Points) Clone() []Point {
other := make([]Point, len(a))
for i, p := range a {
if p == nil {
other[i] = nil
continue
}
switch p := p.(type) {
case *FloatPoint:
other[i] = p.Clone()
case *IntegerPoint:
other[i] = p.Clone()
case *UnsignedPoint:
other[i] = p.Clone()
case *StringPoint:
other[i] = p.Clone()
case *BooleanPoint:
other[i] = p.Clone()
default:
panic(fmt.Sprintf("unable to clone point: %T", p))
}
}
return other
}
// Tags represent a map of keys and values.
// It memoizes its key so it can be used efficiently during query execution.
type Tags struct {
id string
m map[string]string
}
// NewTags returns a new instance of Tags.
func NewTags(m map[string]string) Tags {
if len(m) == 0 {
return Tags{}
}
return Tags{
id: string(encodeTags(m)),
m: m,
}
}
// newTagsID returns a new instance of Tags by parsing the given tag ID.
func newTagsID(id string) Tags {
m := decodeTags([]byte(id))
if len(m) == 0 {
return Tags{}
}
return Tags{id: id, m: m}
}
// Equal compares if the Tags are equal to each other.
func (t Tags) Equal(other Tags) bool {
return t.ID() == other.ID()
}
// ID returns the string identifier for the tags.
func (t Tags) ID() string { return t.id }
// KeyValues returns the underlying map for the tags.
func (t Tags) KeyValues() map[string]string { return t.m }
// Keys returns a sorted list of all keys on the tag.
func (t *Tags) Keys() []string {
if t == nil {
return nil
}
var a []string
for k := range t.m {
a = append(a, k)
}
sort.Strings(a)
return a
}
// Values returns a sorted list of all values on the tag.
func (t *Tags) Values() []string {
if t == nil {
return nil
}
a := make([]string, 0, len(t.m))
for _, v := range t.m {
a = append(a, v)
}
sort.Strings(a)
return a
}
// Value returns the value for a given key.
func (t *Tags) Value(k string) string {
if t == nil {
return ""
}
return t.m[k]
}
// Subset returns a new tags object with a subset of the keys.
func (t *Tags) Subset(keys []string) Tags {
if len(keys) == 0 {
return Tags{}
}
// If keys match existing keys, simply return this tagset.
if keysMatch(t.m, keys) {
return *t
}
// Otherwise create new tag set.
m := make(map[string]string, len(keys))
for _, k := range keys {
m[k] = t.m[k]
}
return NewTags(m)
}
// Equals returns true if t equals other.
func (t *Tags) Equals(other *Tags) bool {
if t == nil && other == nil {
return true
} else if t == nil || other == nil {
return false
}
return t.id == other.id
}
// keysMatch returns true if m has exactly the same keys as listed in keys.
func keysMatch(m map[string]string, keys []string) bool {
if len(keys) != len(m) {
return false
}
for _, k := range keys {
if _, ok := m[k]; !ok {
return false
}
}
return true
}
// encodeTags converts a map of strings to an identifier.
func encodeTags(m map[string]string) []byte {
// Empty maps marshal to empty bytes.
if len(m) == 0 {
return nil
}
// Extract keys and determine final size.
sz := (len(m) * 2) - 1 // separators
keys := make([]string, 0, len(m))
for k, v := range m {
keys = append(keys, k)
sz += len(k) + len(v)
}
sort.Strings(keys)
// Generate marshaled bytes.
b := make([]byte, sz)
buf := b
for _, k := range keys {
copy(buf, k)
buf[len(k)] = '\x00'
buf = buf[len(k)+1:]
}
for i, k := range keys {
v := m[k]
copy(buf, v)
if i < len(keys)-1 {
buf[len(v)] = '\x00'
buf = buf[len(v)+1:]
}
}
return b
}
// decodeTags parses an identifier into a map of tags.
func decodeTags(id []byte) map[string]string {
a := bytes.Split(id, []byte{'\x00'})
// There must be an even number of segments.
if len(a) > 0 && len(a)%2 == 1 {
a = a[:len(a)-1]
}
// Return nil if there are no segments.
if len(a) == 0 {
return nil
}
mid := len(a) / 2
// Decode key/value tags.
m := make(map[string]string)
for i := 0; i < mid; i++ {
m[string(a[i])] = string(a[i+mid])
}
return m
}
func encodeAux(aux []interface{}) []*internal.Aux {
pb := make([]*internal.Aux, len(aux))
for i := range aux {
switch v := aux[i].(type) {
case float64:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Float)), FloatValue: proto.Float64(v)}
case *float64:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Float))}
case int64:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Integer)), IntegerValue: proto.Int64(v)}
case *int64:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Integer))}
case uint64:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Unsigned)), UnsignedValue: proto.Uint64(v)}
case *uint64:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Unsigned))}
case string:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.String)), StringValue: proto.String(v)}
case *string:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.String))}
case bool:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Boolean)), BooleanValue: proto.Bool(v)}
case *bool:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Boolean))}
default:
pb[i] = &internal.Aux{DataType: proto.Int32(int32(influxql.Unknown))}
}
}
return pb
}
func decodeAux(pb []*internal.Aux) []interface{} {
if len(pb) == 0 {
return nil
}
aux := make([]interface{}, len(pb))
for i := range pb {
switch influxql.DataType(pb[i].GetDataType()) {
case influxql.Float:
if pb[i].FloatValue != nil {
aux[i] = *pb[i].FloatValue
} else {
aux[i] = (*float64)(nil)
}
case influxql.Integer:
if pb[i].IntegerValue != nil {
aux[i] = *pb[i].IntegerValue
} else {
aux[i] = (*int64)(nil)
}
case influxql.Unsigned:
if pb[i].UnsignedValue != nil {
aux[i] = *pb[i].UnsignedValue
} else {
aux[i] = (*uint64)(nil)
}
case influxql.String:
if pb[i].StringValue != nil {
aux[i] = *pb[i].StringValue
} else {
aux[i] = (*string)(nil)
}
case influxql.Boolean:
if pb[i].BooleanValue != nil {
aux[i] = *pb[i].BooleanValue
} else {
aux[i] = (*bool)(nil)
}
default:
aux[i] = nil
}
}
return aux
}
func cloneAux(src []interface{}) []interface{} {
if src == nil {
return src
}
dest := make([]interface{}, len(src))
copy(dest, src)
return dest
}
// PointDecoder decodes generic points from a reader.
type PointDecoder struct {
r io.Reader
stats IteratorStats
}
// NewPointDecoder returns a new instance of PointDecoder that reads from r.
func NewPointDecoder(r io.Reader) *PointDecoder {
return &PointDecoder{r: r}
}
// Stats returns iterator stats embedded within the stream.
func (dec *PointDecoder) Stats() IteratorStats { return dec.stats }
// DecodePoint reads from the underlying reader and unmarshals into p.
func (dec *PointDecoder) DecodePoint(p *Point) error {
for {
// Read length.
var sz uint32
if err := binary.Read(dec.r, binary.BigEndian, &sz); err != nil {
return err
}
// Read point data.
buf := make([]byte, sz)
if _, err := io.ReadFull(dec.r, buf); err != nil {
return err
}
// Unmarshal into point.
var pb internal.Point
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
// If the point contains stats then read stats and retry.
if pb.Stats != nil {
dec.stats = decodeIteratorStats(pb.Stats)
continue
}
if pb.IntegerValue != nil {
*p = decodeIntegerPoint(&pb)
} else if pb.UnsignedValue != nil {
*p = decodeUnsignedPoint(&pb)
} else if pb.StringValue != nil {
*p = decodeStringPoint(&pb)
} else if pb.BooleanValue != nil {
*p = decodeBooleanPoint(&pb)
} else {
*p = decodeFloatPoint(&pb)
}
return nil
}
}

187
influxql/query/point_test.go Normal file
View File

@ -0,0 +1,187 @@
package query_test
import (
"reflect"
"strings"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/pkg/deep"
)
func TestPoint_Clone_Float(t *testing.T) {
p := &query.FloatPoint{
Name: "cpu",
Tags: ParseTags("host=server01"),
Time: 5,
Value: 2,
Aux: []interface{}{float64(45)},
}
c := p.Clone()
if p == c {
t.Errorf("clone has the same address as the original: %v == %v", p, c)
}
if !deep.Equal(p, c) {
t.Errorf("mismatched point: %s", spew.Sdump(c))
}
if &p.Aux[0] == &c.Aux[0] {
t.Errorf("aux values share the same address: %v == %v", p.Aux, c.Aux)
} else if !deep.Equal(p.Aux, c.Aux) {
t.Errorf("mismatched aux fields: %v != %v", p.Aux, c.Aux)
}
}
func TestPoint_Clone_Integer(t *testing.T) {
p := &query.IntegerPoint{
Name: "cpu",
Tags: ParseTags("host=server01"),
Time: 5,
Value: 2,
Aux: []interface{}{float64(45)},
}
c := p.Clone()
if p == c {
t.Errorf("clone has the same address as the original: %v == %v", p, c)
}
if !deep.Equal(p, c) {
t.Errorf("mismatched point: %s", spew.Sdump(c))
}
if &p.Aux[0] == &c.Aux[0] {
t.Errorf("aux values share the same address: %v == %v", p.Aux, c.Aux)
} else if !deep.Equal(p.Aux, c.Aux) {
t.Errorf("mismatched aux fields: %v != %v", p.Aux, c.Aux)
}
}
func TestPoint_Clone_String(t *testing.T) {
p := &query.StringPoint{
Name: "cpu",
Tags: ParseTags("host=server01"),
Time: 5,
Value: "clone",
Aux: []interface{}{float64(45)},
}
c := p.Clone()
if p == c {
t.Errorf("clone has the same address as the original: %v == %v", p, c)
}
if !deep.Equal(p, c) {
t.Errorf("mismatched point: %s", spew.Sdump(c))
}
if &p.Aux[0] == &c.Aux[0] {
t.Errorf("aux values share the same address: %v == %v", p.Aux, c.Aux)
} else if !deep.Equal(p.Aux, c.Aux) {
t.Errorf("mismatched aux fields: %v != %v", p.Aux, c.Aux)
}
}
func TestPoint_Clone_Boolean(t *testing.T) {
p := &query.BooleanPoint{
Name: "cpu",
Tags: ParseTags("host=server01"),
Time: 5,
Value: true,
Aux: []interface{}{float64(45)},
}
c := p.Clone()
if p == c {
t.Errorf("clone has the same address as the original: %v == %v", p, c)
}
if !deep.Equal(p, c) {
t.Errorf("mismatched point: %s", spew.Sdump(c))
}
if &p.Aux[0] == &c.Aux[0] {
t.Errorf("aux values share the same address: %v == %v", p.Aux, c.Aux)
} else if !deep.Equal(p.Aux, c.Aux) {
t.Errorf("mismatched aux fields: %v != %v", p.Aux, c.Aux)
}
}
func TestPoint_Clone_Nil(t *testing.T) {
var fp *query.FloatPoint
if p := fp.Clone(); p != nil {
t.Errorf("expected nil, got %v", p)
}
var ip *query.IntegerPoint
if p := ip.Clone(); p != nil {
t.Errorf("expected nil, got %v", p)
}
var sp *query.StringPoint
if p := sp.Clone(); p != nil {
t.Errorf("expected nil, got %v", p)
}
var bp *query.BooleanPoint
if p := bp.Clone(); p != nil {
t.Errorf("expected nil, got %v", p)
}
}
// TestPoint_Fields ensures that no additional fields are added to the point structs.
// This struct is very sensitive and can effect performance unless handled carefully.
// To avoid the struct becoming a dumping ground for every function that needs to store
// miscellaneous information, this test is meant to ensure that new fields don't slip
// into the struct.
func TestPoint_Fields(t *testing.T) {
allowedFields := map[string]bool{
"Name": true,
"Tags": true,
"Time": true,
"Nil": true,
"Value": true,
"Aux": true,
"Aggregated": true,
}
for _, typ := range []reflect.Type{
reflect.TypeOf(query.FloatPoint{}),
reflect.TypeOf(query.IntegerPoint{}),
reflect.TypeOf(query.StringPoint{}),
reflect.TypeOf(query.BooleanPoint{}),
} {
f, ok := typ.FieldByNameFunc(func(name string) bool {
return !allowedFields[name]
})
if ok {
t.Errorf("found an unallowed field in %s: %s %s", typ, f.Name, f.Type)
}
}
}
// Ensure that tags can return a unique id.
func TestTags_ID(t *testing.T) {
tags := query.NewTags(map[string]string{"foo": "bar", "baz": "bat"})
if id := tags.ID(); id != "baz\x00foo\x00bat\x00bar" {
t.Fatalf("unexpected id: %q", id)
}
}
// Ensure that a subset can be created from a tag set.
func TestTags_Subset(t *testing.T) {
tags := query.NewTags(map[string]string{"a": "0", "b": "1", "c": "2"})
subset := tags.Subset([]string{"b", "c", "d"})
if keys := subset.Keys(); !reflect.DeepEqual(keys, []string{"b", "c", "d"}) {
t.Fatalf("unexpected keys: %+v", keys)
} else if v := subset.Value("a"); v != "" {
t.Fatalf("unexpected 'a' value: %s", v)
} else if v := subset.Value("b"); v != "1" {
t.Fatalf("unexpected 'b' value: %s", v)
} else if v := subset.Value("c"); v != "2" {
t.Fatalf("unexpected 'c' value: %s", v)
} else if v := subset.Value("d"); v != "" {
t.Fatalf("unexpected 'd' value: %s", v)
}
}
// ParseTags returns an instance of Tags for a comma-delimited list of key/values.
func ParseTags(s string) query.Tags {
m := make(map[string]string)
for _, kv := range strings.Split(s, ",") {
a := strings.Split(kv, "=")
m[a[0]] = a[1]
}
return query.NewTags(m)
}

7
influxql/query/query.go Normal file
View File

@ -0,0 +1,7 @@
package query // import "github.com/influxdata/influxdb/v2/influxql/query"
//go:generate tmpl -data=@tmpldata iterator.gen.go.tmpl
//go:generate tmpl -data=@tmpldata point.gen.go.tmpl
//go:generate tmpl -data=@tmpldata functions.gen.go.tmpl
//go:generate protoc --gogo_out=. internal/internal.proto

141
influxql/query/result.go Normal file
View File

@ -0,0 +1,141 @@
package query
import (
"encoding/json"
"errors"
"fmt"
"github.com/influxdata/influxdb/v2/v1/models"
"github.com/influxdata/influxql"
)
const (
// WarningLevel is the message level for a warning.
WarningLevel = "warning"
)
// TagSet is a fundamental concept within the query system. It represents a composite series,
// composed of multiple individual series that share a set of tag attributes.
type TagSet struct {
Tags map[string]string
Filters []influxql.Expr
SeriesKeys []string
Key []byte
}
// AddFilter adds a series-level filter to the Tagset.
func (t *TagSet) AddFilter(key string, filter influxql.Expr) {
t.SeriesKeys = append(t.SeriesKeys, key)
t.Filters = append(t.Filters, filter)
}
func (t *TagSet) Len() int { return len(t.SeriesKeys) }
func (t *TagSet) Less(i, j int) bool { return t.SeriesKeys[i] < t.SeriesKeys[j] }
func (t *TagSet) Swap(i, j int) {
t.SeriesKeys[i], t.SeriesKeys[j] = t.SeriesKeys[j], t.SeriesKeys[i]
t.Filters[i], t.Filters[j] = t.Filters[j], t.Filters[i]
}
// Reverse reverses the order of series keys and filters in the TagSet.
func (t *TagSet) Reverse() {
for i, j := 0, len(t.Filters)-1; i < j; i, j = i+1, j-1 {
t.Filters[i], t.Filters[j] = t.Filters[j], t.Filters[i]
t.SeriesKeys[i], t.SeriesKeys[j] = t.SeriesKeys[j], t.SeriesKeys[i]
}
}
// LimitTagSets returns a tag set list with SLIMIT and SOFFSET applied.
func LimitTagSets(a []*TagSet, slimit, soffset int) []*TagSet {
// Ignore if no limit or offset is specified.
if slimit == 0 && soffset == 0 {
return a
}
// If offset is beyond the number of tag sets then return nil.
if soffset > len(a) {
return nil
}
// Clamp limit to the max number of tag sets.
if soffset+slimit > len(a) {
slimit = len(a) - soffset
}
return a[soffset : soffset+slimit]
}
// Message represents a user-facing message to be included with the result.
type Message struct {
Level string `json:"level"`
Text string `json:"text"`
}
// ReadOnlyWarning generates a warning message that tells the user the command
// they are using is being used for writing in a read only context.
//
// This is a temporary method while to be used while transitioning to read only
// operations for issue #6290.
func ReadOnlyWarning(stmt string) *Message {
return &Message{
Level: WarningLevel,
Text: fmt.Sprintf("deprecated use of '%s' in a read only context, please use a POST request instead", stmt),
}
}
// Result represents a resultset returned from a single statement.
// Rows represents a list of rows that can be sorted consistently by name/tag.
type Result struct {
// StatementID is just the statement's position in the query. It's used
// to combine statement results if they're being buffered in memory.
StatementID int
Series models.Rows
Messages []*Message
Partial bool
Err error
}
// MarshalJSON encodes the result into JSON.
func (r *Result) MarshalJSON() ([]byte, error) {
// Define a struct that outputs "error" as a string.
var o struct {
StatementID int `json:"statement_id"`
Series []*models.Row `json:"series,omitempty"`
Messages []*Message `json:"messages,omitempty"`
Partial bool `json:"partial,omitempty"`
Err string `json:"error,omitempty"`
}
// Copy fields to output struct.
o.StatementID = r.StatementID
o.Series = r.Series
o.Messages = r.Messages
o.Partial = r.Partial
if r.Err != nil {
o.Err = r.Err.Error()
}
return json.Marshal(&o)
}
// UnmarshalJSON decodes the data into the Result struct
func (r *Result) UnmarshalJSON(b []byte) error {
var o struct {
StatementID int `json:"statement_id"`
Series []*models.Row `json:"series,omitempty"`
Messages []*Message `json:"messages,omitempty"`
Partial bool `json:"partial,omitempty"`
Err string `json:"error,omitempty"`
}
err := json.Unmarshal(b, &o)
if err != nil {
return err
}
r.StatementID = o.StatementID
r.Series = o.Series
r.Messages = o.Messages
r.Partial = o.Partial
if o.Err != "" {
r.Err = errors.New(o.Err)
}
return nil
}

975
influxql/query/select.go Normal file
View File

@ -0,0 +1,975 @@
package query
import (
"context"
"fmt"
"io"
"sort"
"strings"
"time"
"github.com/influxdata/influxdb/v2/influxql/query/internal/gota"
"github.com/influxdata/influxdb/v2/pkg/tracing"
"github.com/influxdata/influxql"
)
var DefaultTypeMapper = influxql.MultiTypeMapper(
FunctionTypeMapper{},
MathTypeMapper{},
)
// SelectOptions are options that customize the select call.
type SelectOptions struct {
// Authorizer is used to limit access to data
Authorizer Authorizer
// Node to exclusively read from.
// If zero, all nodes are used.
NodeID uint64
// Maximum number of concurrent series.
MaxSeriesN int
// Maximum number of points to read from the query.
// This requires the passed in context to have a Monitor that is
// created using WithMonitor.
MaxPointN int
// Maximum number of buckets for a statement.
MaxBucketsN int
}
// ShardMapper retrieves and maps shards into an IteratorCreator that can later be
// used for executing queries.
type ShardMapper interface {
MapShards(sources influxql.Sources, t influxql.TimeRange, opt SelectOptions) (ShardGroup, error)
}
// ShardGroup represents a shard or a collection of shards that can be accessed
// for creating iterators.
// When creating iterators, the resource used for reading the iterators should be
// separate from the resource used to map the shards. When the ShardGroup is closed,
// it should not close any resources associated with the created Iterator. Those
// resources belong to the Iterator and will be closed when the Iterator itself is
// closed.
// The query engine operates under this assumption and will close the shard group
// after creating the iterators, but before the iterators are actually read.
type ShardGroup interface {
IteratorCreator
influxql.FieldMapper
io.Closer
}
// Select is a prepared statement that is ready to be executed.
type PreparedStatement interface {
// Select creates the Iterators that will be used to read the query.
Select(ctx context.Context) (Cursor, error)
// Explain outputs the explain plan for this statement.
Explain() (string, error)
// Close closes the resources associated with this prepared statement.
// This must be called as the mapped shards may hold open resources such
// as network connections.
Close() error
}
// Prepare will compile the statement with the default compile options and
// then prepare the query.
func Prepare(stmt *influxql.SelectStatement, shardMapper ShardMapper, opt SelectOptions) (PreparedStatement, error) {
c, err := Compile(stmt, CompileOptions{})
if err != nil {
return nil, err
}
return c.Prepare(shardMapper, opt)
}
// Select compiles, prepares, and then initiates execution of the query using the
// default compile options.
func Select(ctx context.Context, stmt *influxql.SelectStatement, shardMapper ShardMapper, opt SelectOptions) (Cursor, error) {
s, err := Prepare(stmt, shardMapper, opt)
if err != nil {
return nil, err
}
// Must be deferred so it runs after Select.
defer s.Close()
return s.Select(ctx)
}
type preparedStatement struct {
stmt *influxql.SelectStatement
opt IteratorOptions
ic interface {
IteratorCreator
io.Closer
}
columns []string
maxPointN int
now time.Time
}
func (p *preparedStatement) Select(ctx context.Context) (Cursor, error) {
// TODO(jsternberg): Remove this hacky method of propagating now.
// Each level of the query should use a time range discovered during
// compilation, but that requires too large of a refactor at the moment.
ctx = context.WithValue(ctx, "now", p.now)
opt := p.opt
opt.InterruptCh = ctx.Done()
cur, err := buildCursor(ctx, p.stmt, p.ic, opt)
if err != nil {
return nil, err
}
// If a monitor exists and we are told there is a maximum number of points,
// register the monitor function.
if m := MonitorFromContext(ctx); m != nil {
if p.maxPointN > 0 {
monitor := PointLimitMonitor(cur, DefaultStatsInterval, p.maxPointN)
m.Monitor(monitor)
}
}
return cur, nil
}
func (p *preparedStatement) Close() error {
return p.ic.Close()
}
// buildExprIterator creates an iterator for an expression.
func buildExprIterator(ctx context.Context, expr influxql.Expr, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions, selector, writeMode bool) (Iterator, error) {
opt.Expr = expr
b := exprIteratorBuilder{
ic: ic,
sources: sources,
opt: opt,
selector: selector,
writeMode: writeMode,
}
switch expr := expr.(type) {
case *influxql.VarRef:
return b.buildVarRefIterator(ctx, expr)
case *influxql.Call:
return b.buildCallIterator(ctx, expr)
default:
return nil, fmt.Errorf("invalid expression type: %T", expr)
}
}
type exprIteratorBuilder struct {
ic IteratorCreator
sources influxql.Sources
opt IteratorOptions
selector bool
writeMode bool
}
func (b *exprIteratorBuilder) buildVarRefIterator(ctx context.Context, expr *influxql.VarRef) (Iterator, error) {
inputs := make([]Iterator, 0, len(b.sources))
if err := func() error {
for _, source := range b.sources {
switch source := source.(type) {
case *influxql.Measurement:
input, err := b.ic.CreateIterator(ctx, source, b.opt)
if err != nil {
return err
}
inputs = append(inputs, input)
case *influxql.SubQuery:
subquery := subqueryBuilder{
ic: b.ic,
stmt: source.Statement,
}
input, err := subquery.buildVarRefIterator(ctx, expr, b.opt)
if err != nil {
return err
} else if input != nil {
inputs = append(inputs, input)
}
}
}
return nil
}(); err != nil {
Iterators(inputs).Close()
return nil, err
}
// Variable references in this section will always go into some call
// iterator. Combine it with a merge iterator.
itr := NewMergeIterator(inputs, b.opt)
if itr == nil {
itr = &nilFloatIterator{}
}
if b.opt.InterruptCh != nil {
itr = NewInterruptIterator(itr, b.opt.InterruptCh)
}
return itr, nil
}
func (b *exprIteratorBuilder) buildCallIterator(ctx context.Context, expr *influxql.Call) (Iterator, error) {
// TODO(jsternberg): Refactor this. This section needs to die in a fire.
opt := b.opt
// Eliminate limits and offsets if they were previously set. These are handled by the caller.
opt.Limit, opt.Offset = 0, 0
switch expr.Name {
case "distinct":
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, b.selector, false)
if err != nil {
return nil, err
}
input, err = NewDistinctIterator(input, opt)
if err != nil {
return nil, err
}
return NewIntervalIterator(input, opt), nil
case "sample":
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false)
if err != nil {
return nil, err
}
size := expr.Args[1].(*influxql.IntegerLiteral)
return newSampleIterator(input, opt, int(size.Val))
case "holt_winters", "holt_winters_with_fit":
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false)
if err != nil {
return nil, err
}
h := expr.Args[1].(*influxql.IntegerLiteral)
m := expr.Args[2].(*influxql.IntegerLiteral)
includeFitData := "holt_winters_with_fit" == expr.Name
interval := opt.Interval.Duration
// Redefine interval to be unbounded to capture all aggregate results
opt.StartTime = influxql.MinTime
opt.EndTime = influxql.MaxTime
opt.Interval = Interval{}
return newHoltWintersIterator(input, opt, int(h.Val), int(m.Val), includeFitData, interval)
case "derivative", "non_negative_derivative", "difference", "non_negative_difference", "moving_average", "exponential_moving_average", "double_exponential_moving_average", "triple_exponential_moving_average", "relative_strength_index", "triple_exponential_derivative", "kaufmans_efficiency_ratio", "kaufmans_adaptive_moving_average", "chande_momentum_oscillator", "elapsed":
if !opt.Interval.IsZero() {
if opt.Ascending {
opt.StartTime -= int64(opt.Interval.Duration)
} else {
opt.EndTime += int64(opt.Interval.Duration)
}
}
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false)
if err != nil {
return nil, err
}
switch expr.Name {
case "derivative", "non_negative_derivative":
interval := opt.DerivativeInterval()
isNonNegative := (expr.Name == "non_negative_derivative")
return newDerivativeIterator(input, opt, interval, isNonNegative)
case "elapsed":
interval := opt.ElapsedInterval()
return newElapsedIterator(input, opt, interval)
case "difference", "non_negative_difference":
isNonNegative := (expr.Name == "non_negative_difference")
return newDifferenceIterator(input, opt, isNonNegative)
case "moving_average":
n := expr.Args[1].(*influxql.IntegerLiteral)
if n.Val > 1 && !opt.Interval.IsZero() {
if opt.Ascending {
opt.StartTime -= int64(opt.Interval.Duration) * (n.Val - 1)
} else {
opt.EndTime += int64(opt.Interval.Duration) * (n.Val - 1)
}
}
return newMovingAverageIterator(input, int(n.Val), opt)
case "exponential_moving_average", "double_exponential_moving_average", "triple_exponential_moving_average", "relative_strength_index", "triple_exponential_derivative":
n := expr.Args[1].(*influxql.IntegerLiteral)
if n.Val > 1 && !opt.Interval.IsZero() {
if opt.Ascending {
opt.StartTime -= int64(opt.Interval.Duration) * (n.Val - 1)
} else {
opt.EndTime += int64(opt.Interval.Duration) * (n.Val - 1)
}
}
nHold := -1
if len(expr.Args) >= 3 {
nHold = int(expr.Args[2].(*influxql.IntegerLiteral).Val)
}
warmupType := gota.WarmEMA
if len(expr.Args) >= 4 {
if warmupType, err = gota.ParseWarmupType(expr.Args[3].(*influxql.StringLiteral).Val); err != nil {
return nil, err
}
}
switch expr.Name {
case "exponential_moving_average":
return newExponentialMovingAverageIterator(input, int(n.Val), nHold, warmupType, opt)
case "double_exponential_moving_average":
return newDoubleExponentialMovingAverageIterator(input, int(n.Val), nHold, warmupType, opt)
case "triple_exponential_moving_average":
return newTripleExponentialMovingAverageIterator(input, int(n.Val), nHold, warmupType, opt)
case "relative_strength_index":
return newRelativeStrengthIndexIterator(input, int(n.Val), nHold, warmupType, opt)
case "triple_exponential_derivative":
return newTripleExponentialDerivativeIterator(input, int(n.Val), nHold, warmupType, opt)
}
case "kaufmans_efficiency_ratio", "kaufmans_adaptive_moving_average":
n := expr.Args[1].(*influxql.IntegerLiteral)
if n.Val > 1 && !opt.Interval.IsZero() {
if opt.Ascending {
opt.StartTime -= int64(opt.Interval.Duration) * (n.Val - 1)
} else {
opt.EndTime += int64(opt.Interval.Duration) * (n.Val - 1)
}
}
nHold := -1
if len(expr.Args) >= 3 {
nHold = int(expr.Args[2].(*influxql.IntegerLiteral).Val)
}
switch expr.Name {
case "kaufmans_efficiency_ratio":
return newKaufmansEfficiencyRatioIterator(input, int(n.Val), nHold, opt)
case "kaufmans_adaptive_moving_average":
return newKaufmansAdaptiveMovingAverageIterator(input, int(n.Val), nHold, opt)
}
case "chande_momentum_oscillator":
n := expr.Args[1].(*influxql.IntegerLiteral)
if n.Val > 1 && !opt.Interval.IsZero() {
if opt.Ascending {
opt.StartTime -= int64(opt.Interval.Duration) * (n.Val - 1)
} else {
opt.EndTime += int64(opt.Interval.Duration) * (n.Val - 1)
}
}
nHold := -1
if len(expr.Args) >= 3 {
nHold = int(expr.Args[2].(*influxql.IntegerLiteral).Val)
}
warmupType := gota.WarmupType(-1)
if len(expr.Args) >= 4 {
wt := expr.Args[3].(*influxql.StringLiteral).Val
if wt != "none" {
if warmupType, err = gota.ParseWarmupType(wt); err != nil {
return nil, err
}
}
}
return newChandeMomentumOscillatorIterator(input, int(n.Val), nHold, warmupType, opt)
}
panic(fmt.Sprintf("invalid series aggregate function: %s", expr.Name))
case "cumulative_sum":
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false)
if err != nil {
return nil, err
}
return newCumulativeSumIterator(input, opt)
case "integral":
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false)
if err != nil {
return nil, err
}
interval := opt.IntegralInterval()
return newIntegralIterator(input, opt, interval)
case "top":
if len(expr.Args) < 2 {
return nil, fmt.Errorf("top() requires 2 or more arguments, got %d", len(expr.Args))
}
var input Iterator
if len(expr.Args) > 2 {
// Create a max iterator using the groupings in the arguments.
dims := make(map[string]struct{}, len(expr.Args)-2+len(opt.GroupBy))
for i := 1; i < len(expr.Args)-1; i++ {
ref := expr.Args[i].(*influxql.VarRef)
dims[ref.Val] = struct{}{}
}
for dim := range opt.GroupBy {
dims[dim] = struct{}{}
}
call := &influxql.Call{
Name: "max",
Args: expr.Args[:1],
}
callOpt := opt
callOpt.Expr = call
callOpt.GroupBy = dims
callOpt.Fill = influxql.NoFill
builder := *b
builder.opt = callOpt
builder.selector = true
builder.writeMode = false
i, err := builder.callIterator(ctx, call, callOpt)
if err != nil {
return nil, err
}
input = i
} else {
// There are no arguments so do not organize the points by tags.
builder := *b
builder.opt.Expr = expr.Args[0]
builder.selector = true
builder.writeMode = false
ref := expr.Args[0].(*influxql.VarRef)
i, err := builder.buildVarRefIterator(ctx, ref)
if err != nil {
return nil, err
}
input = i
}
n := expr.Args[len(expr.Args)-1].(*influxql.IntegerLiteral)
return newTopIterator(input, opt, int(n.Val), b.writeMode)
case "bottom":
if len(expr.Args) < 2 {
return nil, fmt.Errorf("bottom() requires 2 or more arguments, got %d", len(expr.Args))
}
var input Iterator
if len(expr.Args) > 2 {
// Create a max iterator using the groupings in the arguments.
dims := make(map[string]struct{}, len(expr.Args)-2)
for i := 1; i < len(expr.Args)-1; i++ {
ref := expr.Args[i].(*influxql.VarRef)
dims[ref.Val] = struct{}{}
}
for dim := range opt.GroupBy {
dims[dim] = struct{}{}
}
call := &influxql.Call{
Name: "min",
Args: expr.Args[:1],
}
callOpt := opt
callOpt.Expr = call
callOpt.GroupBy = dims
callOpt.Fill = influxql.NoFill
builder := *b
builder.opt = callOpt
builder.selector = true
builder.writeMode = false
i, err := builder.callIterator(ctx, call, callOpt)
if err != nil {
return nil, err
}
input = i
} else {
// There are no arguments so do not organize the points by tags.
builder := *b
builder.opt.Expr = expr.Args[0]
builder.selector = true
builder.writeMode = false
ref := expr.Args[0].(*influxql.VarRef)
i, err := builder.buildVarRefIterator(ctx, ref)
if err != nil {
return nil, err
}
input = i
}
n := expr.Args[len(expr.Args)-1].(*influxql.IntegerLiteral)
return newBottomIterator(input, b.opt, int(n.Val), b.writeMode)
}
itr, err := func() (Iterator, error) {
switch expr.Name {
case "count":
switch arg0 := expr.Args[0].(type) {
case *influxql.Call:
if arg0.Name == "distinct" {
input, err := buildExprIterator(ctx, arg0, b.ic, b.sources, opt, b.selector, false)
if err != nil {
return nil, err
}
return newCountIterator(input, opt)
}
}
fallthrough
case "min", "max", "sum", "first", "last", "mean":
return b.callIterator(ctx, expr, opt)
case "median":
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false)
if err != nil {
return nil, err
}
return newMedianIterator(input, opt)
case "mode":
input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false)
if err != nil {
return nil, err
}
return NewModeIterator(input, opt)
case "stddev":
input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false)
if err != nil {
return nil, err
}
return newStddevIterator(input, opt)
case "spread":
// OPTIMIZE(benbjohnson): convert to map/reduce
input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false)
if err != nil {
return nil, err
}
return newSpreadIterator(input, opt)
case "percentile":
opt.Ordered = true
input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false)
if err != nil {
return nil, err
}
var percentile float64
switch arg := expr.Args[1].(type) {
case *influxql.NumberLiteral:
percentile = arg.Val
case *influxql.IntegerLiteral:
percentile = float64(arg.Val)
}
return newPercentileIterator(input, opt, percentile)
default:
return nil, fmt.Errorf("unsupported call: %s", expr.Name)
}
}()
if err != nil {
return nil, err
}
if !b.selector || !opt.Interval.IsZero() {
itr = NewIntervalIterator(itr, opt)
if !opt.Interval.IsZero() && opt.Fill != influxql.NoFill {
itr = NewFillIterator(itr, expr, opt)
}
}
if opt.InterruptCh != nil {
itr = NewInterruptIterator(itr, opt.InterruptCh)
}
return itr, nil
}
func (b *exprIteratorBuilder) callIterator(ctx context.Context, expr *influxql.Call, opt IteratorOptions) (Iterator, error) {
inputs := make([]Iterator, 0, len(b.sources))
if err := func() error {
for _, source := range b.sources {
switch source := source.(type) {
case *influxql.Measurement:
input, err := b.ic.CreateIterator(ctx, source, opt)
if err != nil {
return err
}
inputs = append(inputs, input)
case *influxql.SubQuery:
// Identify the name of the field we are using.
arg0 := expr.Args[0].(*influxql.VarRef)
opt.Ordered = false
input, err := buildExprIterator(ctx, arg0, b.ic, []influxql.Source{source}, opt, b.selector, false)
if err != nil {
return err
}
// Wrap the result in a call iterator.
i, err := NewCallIterator(input, opt)
if err != nil {
input.Close()
return err
}
inputs = append(inputs, i)
}
}
return nil
}(); err != nil {
Iterators(inputs).Close()
return nil, err
}
itr, err := Iterators(inputs).Merge(opt)
if err != nil {
Iterators(inputs).Close()
return nil, err
} else if itr == nil {
itr = &nilFloatIterator{}
}
return itr, nil
}
func buildCursor(ctx context.Context, stmt *influxql.SelectStatement, ic IteratorCreator, opt IteratorOptions) (Cursor, error) {
span := tracing.SpanFromContext(ctx)
if span != nil {
span = span.StartSpan("build_cursor")
defer span.Finish()
span.SetLabels("statement", stmt.String())
ctx = tracing.NewContextWithSpan(ctx, span)
}
switch opt.Fill {
case influxql.NumberFill:
if v, ok := opt.FillValue.(int); ok {
opt.FillValue = int64(v)
}
case influxql.PreviousFill:
opt.FillValue = SkipDefault
}
fields := make([]*influxql.Field, 0, len(stmt.Fields)+1)
if !stmt.OmitTime {
// Add a field with the variable "time" if we have not omitted time.
fields = append(fields, &influxql.Field{
Expr: &influxql.VarRef{
Val: "time",
Type: influxql.Time,
},
})
}
// Iterate through each of the fields to add them to the value mapper.
valueMapper := newValueMapper()
for _, f := range stmt.Fields {
fields = append(fields, valueMapper.Map(f))
// If the field is a top() or bottom() call, we need to also add
// the extra variables if we are not writing into a target.
if stmt.Target != nil {
continue
}
switch expr := f.Expr.(type) {
case *influxql.Call:
if expr.Name == "top" || expr.Name == "bottom" {
for i := 1; i < len(expr.Args)-1; i++ {
nf := influxql.Field{Expr: expr.Args[i]}
fields = append(fields, valueMapper.Map(&nf))
}
}
}
}
// Set the aliases on each of the columns to what the final name should be.
columns := stmt.ColumnNames()
for i, f := range fields {
f.Alias = columns[i]
}
// Retrieve the refs to retrieve the auxiliary fields.
var auxKeys []influxql.VarRef
if len(valueMapper.refs) > 0 {
opt.Aux = make([]influxql.VarRef, 0, len(valueMapper.refs))
for ref := range valueMapper.refs {
opt.Aux = append(opt.Aux, *ref)
}
sort.Sort(influxql.VarRefs(opt.Aux))
auxKeys = make([]influxql.VarRef, len(opt.Aux))
for i, ref := range opt.Aux {
auxKeys[i] = valueMapper.symbols[ref.String()]
}
}
// If there are no calls, then produce an auxiliary cursor.
if len(valueMapper.calls) == 0 {
// If all of the auxiliary keys are of an unknown type,
// do not construct the iterator and return a null cursor.
if !hasValidType(auxKeys) {
return newNullCursor(fields), nil
}
itr, err := buildAuxIterator(ctx, ic, stmt.Sources, opt)
if err != nil {
return nil, err
}
// Create a slice with an empty first element.
keys := []influxql.VarRef{{}}
keys = append(keys, auxKeys...)
scanner := NewIteratorScanner(itr, keys, opt.FillValue)
return newScannerCursor(scanner, fields, opt), nil
}
// Check to see if this is a selector statement.
// It is a selector if it is the only selector call and the call itself
// is a selector.
selector := len(valueMapper.calls) == 1
if selector {
for call := range valueMapper.calls {
if !influxql.IsSelector(call) {
selector = false
}
}
}
// Produce an iterator for every single call and create an iterator scanner
// associated with it.
scanners := make([]IteratorScanner, 0, len(valueMapper.calls))
for call := range valueMapper.calls {
driver := valueMapper.table[call]
if driver.Type == influxql.Unknown {
// The primary driver of this call is of unknown type, so skip this.
continue
}
itr, err := buildFieldIterator(ctx, call, ic, stmt.Sources, opt, selector, stmt.Target != nil)
if err != nil {
for _, s := range scanners {
s.Close()
}
return nil, err
}
keys := make([]influxql.VarRef, 0, len(auxKeys)+1)
keys = append(keys, driver)
keys = append(keys, auxKeys...)
scanner := NewIteratorScanner(itr, keys, opt.FillValue)
scanners = append(scanners, scanner)
}
if len(scanners) == 0 {
return newNullCursor(fields), nil
} else if len(scanners) == 1 {
return newScannerCursor(scanners[0], fields, opt), nil
}
return newMultiScannerCursor(scanners, fields, opt), nil
}
func buildAuxIterator(ctx context.Context, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions) (Iterator, error) {
span := tracing.SpanFromContext(ctx)
if span != nil {
span = span.StartSpan("iterator_scanner")
defer span.Finish()
auxFieldNames := make([]string, len(opt.Aux))
for i, ref := range opt.Aux {
auxFieldNames[i] = ref.String()
}
span.SetLabels("auxiliary_fields", strings.Join(auxFieldNames, ", "))
ctx = tracing.NewContextWithSpan(ctx, span)
}
inputs := make([]Iterator, 0, len(sources))
if err := func() error {
for _, source := range sources {
switch source := source.(type) {
case *influxql.Measurement:
input, err := ic.CreateIterator(ctx, source, opt)
if err != nil {
return err
}
inputs = append(inputs, input)
case *influxql.SubQuery:
b := subqueryBuilder{
ic: ic,
stmt: source.Statement,
}
input, err := b.buildAuxIterator(ctx, opt)
if err != nil {
return err
} else if input != nil {
inputs = append(inputs, input)
}
}
}
return nil
}(); err != nil {
Iterators(inputs).Close()
return nil, err
}
// Merge iterators to read auxilary fields.
input, err := Iterators(inputs).Merge(opt)
if err != nil {
Iterators(inputs).Close()
return nil, err
} else if input == nil {
input = &nilFloatIterator{}
}
// Filter out duplicate rows, if required.
if opt.Dedupe {
// If there is no group by and it is a float iterator, see if we can use a fast dedupe.
if itr, ok := input.(FloatIterator); ok && len(opt.Dimensions) == 0 {
if sz := len(opt.Aux); sz > 0 && sz < 3 {
input = newFloatFastDedupeIterator(itr)
} else {
input = NewDedupeIterator(itr)
}
} else {
input = NewDedupeIterator(input)
}
}
// Apply limit & offset.
if opt.Limit > 0 || opt.Offset > 0 {
input = NewLimitIterator(input, opt)
}
return input, nil
}
func buildFieldIterator(ctx context.Context, expr influxql.Expr, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions, selector, writeMode bool) (Iterator, error) {
span := tracing.SpanFromContext(ctx)
if span != nil {
span = span.StartSpan("iterator_scanner")
defer span.Finish()
labels := []string{"expr", expr.String()}
if len(opt.Aux) > 0 {
auxFieldNames := make([]string, len(opt.Aux))
for i, ref := range opt.Aux {
auxFieldNames[i] = ref.String()
}
labels = append(labels, "auxiliary_fields", strings.Join(auxFieldNames, ", "))
}
span.SetLabels(labels...)
ctx = tracing.NewContextWithSpan(ctx, span)
}
input, err := buildExprIterator(ctx, expr, ic, sources, opt, selector, writeMode)
if err != nil {
return nil, err
}
// Apply limit & offset.
if opt.Limit > 0 || opt.Offset > 0 {
input = NewLimitIterator(input, opt)
}
return input, nil
}
type valueMapper struct {
// An index that maps a node's string output to its symbol so that all
// nodes with the same signature are mapped the same.
symbols map[string]influxql.VarRef
// An index that maps a specific expression to a symbol. This ensures that
// only expressions that were mapped get symbolized.
table map[influxql.Expr]influxql.VarRef
// A collection of all of the calls in the table.
calls map[*influxql.Call]struct{}
// A collection of all of the calls in the table.
refs map[*influxql.VarRef]struct{}
i int
}
func newValueMapper() *valueMapper {
return &valueMapper{
symbols: make(map[string]influxql.VarRef),
table: make(map[influxql.Expr]influxql.VarRef),
calls: make(map[*influxql.Call]struct{}),
refs: make(map[*influxql.VarRef]struct{}),
}
}
func (v *valueMapper) Map(field *influxql.Field) *influxql.Field {
clone := *field
clone.Expr = influxql.CloneExpr(field.Expr)
influxql.Walk(v, clone.Expr)
clone.Expr = influxql.RewriteExpr(clone.Expr, v.rewriteExpr)
return &clone
}
func (v *valueMapper) Visit(n influxql.Node) influxql.Visitor {
expr, ok := n.(influxql.Expr)
if !ok {
return v
}
key := expr.String()
symbol, ok := v.symbols[key]
if !ok {
// This symbol has not been assigned yet.
// If this is a call or expression, mark the node
// as stored in the symbol table.
switch n := n.(type) {
case *influxql.Call:
if isMathFunction(n) {
return v
}
v.calls[n] = struct{}{}
case *influxql.VarRef:
v.refs[n] = struct{}{}
default:
return v
}
// Determine the symbol name and the symbol type.
symbolName := fmt.Sprintf("val%d", v.i)
valuer := influxql.TypeValuerEval{
TypeMapper: DefaultTypeMapper,
}
typ, _ := valuer.EvalType(expr)
symbol = influxql.VarRef{
Val: symbolName,
Type: typ,
}
// Assign this symbol to the symbol table if it is not presently there
// and increment the value index number.
v.symbols[key] = symbol
v.i++
}
// Store the symbol for this expression so we can later rewrite
// the query correctly.
v.table[expr] = symbol
return nil
}
func (v *valueMapper) rewriteExpr(expr influxql.Expr) influxql.Expr {
symbol, ok := v.table[expr]
if !ok {
return expr
}
return &symbol
}
func validateTypes(stmt *influxql.SelectStatement) error {
valuer := influxql.TypeValuerEval{
TypeMapper: influxql.MultiTypeMapper(
FunctionTypeMapper{},
MathTypeMapper{},
),
}
for _, f := range stmt.Fields {
if _, err := valuer.EvalType(f.Expr); err != nil {
return err
}
}
return nil
}
// hasValidType returns true if there is at least one non-unknown type
// in the slice.
func hasValidType(refs []influxql.VarRef) bool {
for _, ref := range refs {
if ref.Type != influxql.Unknown {
return true
}
}
return false
}

4225
influxql/query/select_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

496
influxql/query/statement_rewriter.go Normal file
View File

@ -0,0 +1,496 @@
package query
import (
"errors"
"regexp"
"github.com/influxdata/influxql"
)
// RewriteStatement rewrites stmt into a new statement, if applicable.
func RewriteStatement(stmt influxql.Statement) (influxql.Statement, error) {
switch stmt := stmt.(type) {
case *influxql.ShowFieldKeysStatement:
return rewriteShowFieldKeysStatement(stmt)
case *influxql.ShowFieldKeyCardinalityStatement:
return rewriteShowFieldKeyCardinalityStatement(stmt)
case *influxql.ShowMeasurementsStatement:
return rewriteShowMeasurementsStatement(stmt)
case *influxql.ShowMeasurementCardinalityStatement:
return rewriteShowMeasurementCardinalityStatement(stmt)
case *influxql.ShowSeriesStatement:
return rewriteShowSeriesStatement(stmt)
case *influxql.ShowSeriesCardinalityStatement:
return rewriteShowSeriesCardinalityStatement(stmt)
case *influxql.ShowTagKeysStatement:
return rewriteShowTagKeysStatement(stmt)
case *influxql.ShowTagKeyCardinalityStatement:
return rewriteShowTagKeyCardinalityStatement(stmt)
case *influxql.ShowTagValuesStatement:
return rewriteShowTagValuesStatement(stmt)
case *influxql.ShowTagValuesCardinalityStatement:
return rewriteShowTagValuesCardinalityStatement(stmt)
default:
return stmt, nil
}
}
func rewriteShowFieldKeysStatement(stmt *influxql.ShowFieldKeysStatement) (influxql.Statement, error) {
return &influxql.SelectStatement{
Fields: influxql.Fields([]*influxql.Field{
{Expr: &influxql.VarRef{Val: "fieldKey"}},
{Expr: &influxql.VarRef{Val: "fieldType"}},
}),
Sources: rewriteSources(stmt.Sources, "_fieldKeys", stmt.Database),
Condition: rewriteSourcesCondition(stmt.Sources, nil),
Offset: stmt.Offset,
Limit: stmt.Limit,
SortFields: stmt.SortFields,
OmitTime: true,
Dedupe: true,
IsRawQuery: true,
}, nil
}
func rewriteShowFieldKeyCardinalityStatement(stmt *influxql.ShowFieldKeyCardinalityStatement) (influxql.Statement, error) {
// Check for time in WHERE clause (not supported).
if influxql.HasTimeExpr(stmt.Condition) {
return nil, errors.New("SHOW FIELD KEY CARDINALITY doesn't support time in WHERE clause")
}
// Use all field keys, if zero.
if len(stmt.Sources) == 0 {
stmt.Sources = influxql.Sources{
&influxql.Measurement{Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`.+`)}},
}
}
return &influxql.SelectStatement{
Fields: []*influxql.Field{
{
Expr: &influxql.Call{
Name: "count",
Args: []influxql.Expr{
&influxql.Call{
Name: "distinct",
Args: []influxql.Expr{&influxql.VarRef{Val: "_fieldKey"}},
},
},
},
Alias: "count",
},
},
Sources: rewriteSources2(stmt.Sources, stmt.Database),
Condition: stmt.Condition,
Dimensions: stmt.Dimensions,
Offset: stmt.Offset,
Limit: stmt.Limit,
OmitTime: true,
}, nil
}
func rewriteShowMeasurementsStatement(stmt *influxql.ShowMeasurementsStatement) (influxql.Statement, error) {
var sources influxql.Sources
if stmt.Source != nil {
sources = influxql.Sources{stmt.Source}
}
// Currently time based SHOW MEASUREMENT queries can't be supported because
// it's not possible to appropriate set operations such as a negated regex
// using the query engine.
if influxql.HasTimeExpr(stmt.Condition) {
return nil, errors.New("SHOW MEASUREMENTS doesn't support time in WHERE clause")
}
// rewrite condition to push a source measurement into a "_name" tag.
stmt.Condition = rewriteSourcesCondition(sources, stmt.Condition)
return stmt, nil
}
func rewriteShowMeasurementCardinalityStatement(stmt *influxql.ShowMeasurementCardinalityStatement) (influxql.Statement, error) {
// TODO(edd): currently we only support cardinality estimation for certain
// types of query. As the estimation coverage is expanded, this condition
// will become less strict.
if !stmt.Exact && stmt.Sources == nil && stmt.Condition == nil && stmt.Dimensions == nil && stmt.Limit == 0 && stmt.Offset == 0 {
return stmt, nil
}
// Check for time in WHERE clause (not supported).
if influxql.HasTimeExpr(stmt.Condition) {
return nil, errors.New("SHOW MEASUREMENT EXACT CARDINALITY doesn't support time in WHERE clause")
}
// Use all measurements, if zero.
if len(stmt.Sources) == 0 {
stmt.Sources = influxql.Sources{
&influxql.Measurement{Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`.+`)}},
}
}
return &influxql.SelectStatement{
Fields: []*influxql.Field{
{
Expr: &influxql.Call{
Name: "count",
Args: []influxql.Expr{
&influxql.Call{
Name: "distinct",
Args: []influxql.Expr{&influxql.VarRef{Val: "_name"}},
},
},
},
Alias: "count",
},
},
Sources: rewriteSources2(stmt.Sources, stmt.Database),
Condition: stmt.Condition,
Dimensions: stmt.Dimensions,
Offset: stmt.Offset,
Limit: stmt.Limit,
OmitTime: true,
StripName: true,
}, nil
}
func rewriteShowSeriesStatement(stmt *influxql.ShowSeriesStatement) (influxql.Statement, error) {
s := &influxql.SelectStatement{
Condition: stmt.Condition,
Offset: stmt.Offset,
Limit: stmt.Limit,
SortFields: stmt.SortFields,
OmitTime: true,
StripName: true,
Dedupe: true,
IsRawQuery: true,
}
// Check if we can exclusively use the index.
if !influxql.HasTimeExpr(stmt.Condition) {
s.Fields = []*influxql.Field{{Expr: &influxql.VarRef{Val: "key"}}}
s.Sources = rewriteSources(stmt.Sources, "_series", stmt.Database)
s.Condition = rewriteSourcesCondition(s.Sources, s.Condition)
return s, nil
}
// The query is bounded by time then it will have to query TSM data rather
// than utilising the index via system iterators.
s.Fields = []*influxql.Field{
{Expr: &influxql.VarRef{Val: "_seriesKey"}, Alias: "key"},
}
s.Sources = rewriteSources2(stmt.Sources, stmt.Database)
return s, nil
}
func rewriteShowSeriesCardinalityStatement(stmt *influxql.ShowSeriesCardinalityStatement) (influxql.Statement, error) {
// TODO(edd): currently we only support cardinality estimation for certain
// types of query. As the estimation coverage is expanded, this condition
// will become less strict.
if !stmt.Exact && stmt.Sources == nil && stmt.Condition == nil && stmt.Dimensions == nil && stmt.Limit == 0 && stmt.Offset == 0 {
return stmt, nil
}
// Check for time in WHERE clause (not supported).
if influxql.HasTimeExpr(stmt.Condition) {
return nil, errors.New("SHOW SERIES EXACT CARDINALITY doesn't support time in WHERE clause")
}
// Use all measurements, if zero.
if len(stmt.Sources) == 0 {
stmt.Sources = influxql.Sources{
&influxql.Measurement{Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`.+`)}},
}
}
return &influxql.SelectStatement{
Fields: []*influxql.Field{
{
Expr: &influxql.Call{
Name: "count",
Args: []influxql.Expr{&influxql.Call{
Name: "distinct",
Args: []influxql.Expr{&influxql.VarRef{Val: "_seriesKey"}},
}},
},
Alias: "count",
},
},
Sources: rewriteSources2(stmt.Sources, stmt.Database),
Condition: stmt.Condition,
Dimensions: stmt.Dimensions,
Offset: stmt.Offset,
Limit: stmt.Limit,
OmitTime: true,
}, nil
}
func rewriteShowTagValuesStatement(stmt *influxql.ShowTagValuesStatement) (influxql.Statement, error) {
var expr influxql.Expr
if list, ok := stmt.TagKeyExpr.(*influxql.ListLiteral); ok {
for _, tagKey := range list.Vals {
tagExpr := &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "_tagKey"},
RHS: &influxql.StringLiteral{Val: tagKey},
}
if expr != nil {
expr = &influxql.BinaryExpr{
Op: influxql.OR,
LHS: expr,
RHS: tagExpr,
}
} else {
expr = tagExpr
}
}
} else {
expr = &influxql.BinaryExpr{
Op: stmt.Op,
LHS: &influxql.VarRef{Val: "_tagKey"},
RHS: stmt.TagKeyExpr,
}
}
// Set condition or "AND" together.
condition := stmt.Condition
if condition == nil {
condition = expr
} else {
condition = &influxql.BinaryExpr{
Op: influxql.AND,
LHS: &influxql.ParenExpr{Expr: condition},
RHS: &influxql.ParenExpr{Expr: expr},
}
}
condition = rewriteSourcesCondition(stmt.Sources, condition)
return &influxql.ShowTagValuesStatement{
Database: stmt.Database,
Op: stmt.Op,
TagKeyExpr: stmt.TagKeyExpr,
Condition: condition,
SortFields: stmt.SortFields,
Limit: stmt.Limit,
Offset: stmt.Offset,
}, nil
}
func rewriteShowTagValuesCardinalityStatement(stmt *influxql.ShowTagValuesCardinalityStatement) (influxql.Statement, error) {
// Use all measurements, if zero.
if len(stmt.Sources) == 0 {
stmt.Sources = influxql.Sources{
&influxql.Measurement{Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`.+`)}},
}
}
var expr influxql.Expr
if list, ok := stmt.TagKeyExpr.(*influxql.ListLiteral); ok {
for _, tagKey := range list.Vals {
tagExpr := &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "_tagKey"},
RHS: &influxql.StringLiteral{Val: tagKey},
}
if expr != nil {
expr = &influxql.BinaryExpr{
Op: influxql.OR,
LHS: expr,
RHS: tagExpr,
}
} else {
expr = tagExpr
}
}
} else {
expr = &influxql.BinaryExpr{
Op: stmt.Op,
LHS: &influxql.VarRef{Val: "_tagKey"},
RHS: stmt.TagKeyExpr,
}
}
// Set condition or "AND" together.
condition := stmt.Condition
if condition == nil {
condition = expr
} else {
condition = &influxql.BinaryExpr{
Op: influxql.AND,
LHS: &influxql.ParenExpr{Expr: condition},
RHS: &influxql.ParenExpr{Expr: expr},
}
}
return &influxql.SelectStatement{
Fields: []*influxql.Field{
{
Expr: &influxql.Call{
Name: "count",
Args: []influxql.Expr{
&influxql.Call{
Name: "distinct",
Args: []influxql.Expr{&influxql.VarRef{Val: "_tagValue"}},
},
},
},
Alias: "count",
},
},
Sources: rewriteSources2(stmt.Sources, stmt.Database),
Condition: condition,
Dimensions: stmt.Dimensions,
Offset: stmt.Offset,
Limit: stmt.Limit,
OmitTime: true,
}, nil
}
func rewriteShowTagKeysStatement(stmt *influxql.ShowTagKeysStatement) (influxql.Statement, error) {
return &influxql.ShowTagKeysStatement{
Database: stmt.Database,
Condition: rewriteSourcesCondition(stmt.Sources, stmt.Condition),
SortFields: stmt.SortFields,
Limit: stmt.Limit,
Offset: stmt.Offset,
SLimit: stmt.SLimit,
SOffset: stmt.SOffset,
}, nil
}
func rewriteShowTagKeyCardinalityStatement(stmt *influxql.ShowTagKeyCardinalityStatement) (influxql.Statement, error) {
// Check for time in WHERE clause (not supported).
if influxql.HasTimeExpr(stmt.Condition) {
return nil, errors.New("SHOW TAG KEY EXACT CARDINALITY doesn't support time in WHERE clause")
}
// Use all measurements, if zero.
if len(stmt.Sources) == 0 {
stmt.Sources = influxql.Sources{
&influxql.Measurement{Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`.+`)}},
}
}
return &influxql.SelectStatement{
Fields: []*influxql.Field{
{
Expr: &influxql.Call{
Name: "count",
Args: []influxql.Expr{
&influxql.Call{
Name: "distinct",
Args: []influxql.Expr{&influxql.VarRef{Val: "_tagKey"}},
},
},
},
Alias: "count",
},
},
Sources: rewriteSources2(stmt.Sources, stmt.Database),
Condition: stmt.Condition,
Dimensions: stmt.Dimensions,
Offset: stmt.Offset,
Limit: stmt.Limit,
OmitTime: true,
}, nil
}
// rewriteSources rewrites sources to include the provided system iterator.
//
// rewriteSources also sets the default database where necessary.
func rewriteSources(sources influxql.Sources, systemIterator, defaultDatabase string) influxql.Sources {
newSources := influxql.Sources{}
for _, src := range sources {
if src == nil {
continue
}
mm := src.(*influxql.Measurement)
database := mm.Database
if database == "" {
database = defaultDatabase
}
newM := mm.Clone()
newM.SystemIterator, newM.Database = systemIterator, database
newSources = append(newSources, newM)
}
if len(newSources) <= 0 {
return append(newSources, &influxql.Measurement{
Database: defaultDatabase,
SystemIterator: systemIterator,
})
}
return newSources
}
// rewriteSourcesCondition rewrites sources into `name` expressions.
// Merges with cond and returns a new condition.
func rewriteSourcesCondition(sources influxql.Sources, cond influxql.Expr) influxql.Expr {
if len(sources) == 0 {
return cond
}
// Generate an OR'd set of filters on source name.
var scond influxql.Expr
for _, source := range sources {
mm := source.(*influxql.Measurement)
// Generate a filtering expression on the measurement name.
var expr influxql.Expr
if mm.Regex != nil {
expr = &influxql.BinaryExpr{
Op: influxql.EQREGEX,
LHS: &influxql.VarRef{Val: "_name"},
RHS: &influxql.RegexLiteral{Val: mm.Regex.Val},
}
} else if mm.Name != "" {
expr = &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "_name"},
RHS: &influxql.StringLiteral{Val: mm.Name},
}
}
if scond == nil {
scond = expr
} else {
scond = &influxql.BinaryExpr{
Op: influxql.OR,
LHS: scond,
RHS: expr,
}
}
}
// This is the case where the original query has a WHERE on a tag, and also
// is requesting from a specific source.
if cond != nil && scond != nil {
return &influxql.BinaryExpr{
Op: influxql.AND,
LHS: &influxql.ParenExpr{Expr: scond},
RHS: &influxql.ParenExpr{Expr: cond},
}
} else if cond != nil {
// This is the case where the original query has a WHERE on a tag but
// is not requesting from a specific source.
return cond
}
return scond
}
func rewriteSources2(sources influxql.Sources, database string) influxql.Sources {
if len(sources) == 0 {
sources = influxql.Sources{&influxql.Measurement{Regex: &influxql.RegexLiteral{Val: matchAllRegex.Copy()}}}
}
for _, source := range sources {
switch source := source.(type) {
case *influxql.Measurement:
if source.Database == "" {
source.Database = database
}
}
}
return sources
}
var matchAllRegex = regexp.MustCompile(`.+`)

320
influxql/query/statement_rewriter_test.go Normal file
View File

@ -0,0 +1,320 @@
package query_test
import (
"testing"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxql"
)
func TestRewriteStatement(t *testing.T) {
tests := []struct {
stmt string
s string
}{
{
stmt: `SHOW FIELD KEYS`,
s: `SELECT fieldKey, fieldType FROM _fieldKeys`,
},
{
stmt: `SHOW FIELD KEYS ON db0`,
s: `SELECT fieldKey, fieldType FROM db0.._fieldKeys`,
},
{
stmt: `SHOW FIELD KEYS FROM cpu`,
s: `SELECT fieldKey, fieldType FROM _fieldKeys WHERE _name = 'cpu'`,
},
{
stmt: `SHOW FIELD KEYS ON db0 FROM cpu`,
s: `SELECT fieldKey, fieldType FROM db0.._fieldKeys WHERE _name = 'cpu'`,
},
{
stmt: `SHOW FIELD KEYS FROM /c.*/`,
s: `SELECT fieldKey, fieldType FROM _fieldKeys WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW FIELD KEYS ON db0 FROM /c.*/`,
s: `SELECT fieldKey, fieldType FROM db0.._fieldKeys WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW FIELD KEYS FROM mydb.myrp2.cpu`,
s: `SELECT fieldKey, fieldType FROM mydb.myrp2._fieldKeys WHERE _name = 'cpu'`,
},
{
stmt: `SHOW FIELD KEYS ON db0 FROM mydb.myrp2.cpu`,
s: `SELECT fieldKey, fieldType FROM mydb.myrp2._fieldKeys WHERE _name = 'cpu'`,
},
{
stmt: `SHOW FIELD KEYS FROM mydb.myrp2./c.*/`,
s: `SELECT fieldKey, fieldType FROM mydb.myrp2._fieldKeys WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW FIELD KEYS ON db0 FROM mydb.myrp2./c.*/`,
s: `SELECT fieldKey, fieldType FROM mydb.myrp2._fieldKeys WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW SERIES`,
s: `SELECT "key" FROM _series`,
},
{
stmt: `SHOW SERIES ON db0`,
s: `SELECT "key" FROM db0.._series`,
},
{
stmt: `SHOW SERIES FROM cpu`,
s: `SELECT "key" FROM _series WHERE _name = 'cpu'`,
},
{
stmt: `SHOW SERIES ON db0 FROM cpu`,
s: `SELECT "key" FROM db0.._series WHERE _name = 'cpu'`,
},
{
stmt: `SHOW SERIES FROM mydb.myrp1.cpu`,
s: `SELECT "key" FROM mydb.myrp1._series WHERE _name = 'cpu'`,
},
{
stmt: `SHOW SERIES ON db0 FROM mydb.myrp1.cpu`,
s: `SELECT "key" FROM mydb.myrp1._series WHERE _name = 'cpu'`,
},
{
stmt: `SHOW SERIES FROM mydb.myrp1./c.*/`,
s: `SELECT "key" FROM mydb.myrp1._series WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW SERIES FROM mydb.myrp1./c.*/ WHERE region = 'uswest'`,
s: `SELECT "key" FROM mydb.myrp1._series WHERE (_name =~ /c.*/) AND (region = 'uswest')`,
},
{
stmt: `SHOW SERIES ON db0 FROM mydb.myrp1./c.*/`,
s: `SELECT "key" FROM mydb.myrp1._series WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW SERIES WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM /.+/ WHERE time > 0`,
},
{
stmt: `SHOW SERIES ON db0 WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM db0../.+/ WHERE time > 0`,
},
{
stmt: `SHOW SERIES FROM cpu WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM cpu WHERE time > 0`,
},
{
stmt: `SHOW SERIES ON db0 FROM cpu WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM db0..cpu WHERE time > 0`,
},
{
stmt: `SHOW SERIES FROM mydb.myrp1.cpu WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM mydb.myrp1.cpu WHERE time > 0`,
},
{
stmt: `SHOW SERIES ON db0 FROM mydb.myrp1.cpu WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM mydb.myrp1.cpu WHERE time > 0`,
},
{
stmt: `SHOW SERIES FROM mydb.myrp1./c.*/ WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM mydb.myrp1./c.*/ WHERE time > 0`,
},
{
stmt: `SHOW SERIES FROM mydb.myrp1./c.*/ WHERE region = 'uswest' AND time > 0`,
s: `SELECT _seriesKey AS "key" FROM mydb.myrp1./c.*/ WHERE region = 'uswest' AND time > 0`,
},
{
stmt: `SHOW SERIES ON db0 FROM mydb.myrp1./c.*/ WHERE time > 0`,
s: `SELECT _seriesKey AS "key" FROM mydb.myrp1./c.*/ WHERE time > 0`,
},
{
stmt: `SHOW SERIES CARDINALITY FROM m`,
s: `SELECT count(distinct(_seriesKey)) AS count FROM m`,
},
{
stmt: `SHOW SERIES EXACT CARDINALITY`,
s: `SELECT count(distinct(_seriesKey)) AS count FROM /.+/`,
},
{
stmt: `SHOW SERIES EXACT CARDINALITY FROM m`,
s: `SELECT count(distinct(_seriesKey)) AS count FROM m`,
},
{
stmt: `SHOW TAG KEYS`,
s: `SHOW TAG KEYS`,
},
{
stmt: `SHOW TAG KEYS ON db0`,
s: `SHOW TAG KEYS ON db0`,
},
{
stmt: `SHOW TAG KEYS FROM cpu`,
s: `SHOW TAG KEYS WHERE _name = 'cpu'`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM cpu`,
s: `SHOW TAG KEYS ON db0 WHERE _name = 'cpu'`,
},
{
stmt: `SHOW TAG KEYS FROM /c.*/`,
s: `SHOW TAG KEYS WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM /c.*/`,
s: `SHOW TAG KEYS ON db0 WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW TAG KEYS FROM cpu WHERE region = 'uswest'`,
s: `SHOW TAG KEYS WHERE (_name = 'cpu') AND (region = 'uswest')`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM cpu WHERE region = 'uswest'`,
s: `SHOW TAG KEYS ON db0 WHERE (_name = 'cpu') AND (region = 'uswest')`,
},
{
stmt: `SHOW TAG KEYS FROM mydb.myrp1.cpu`,
s: `SHOW TAG KEYS WHERE _name = 'cpu'`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM mydb.myrp1.cpu`,
s: `SHOW TAG KEYS ON db0 WHERE _name = 'cpu'`,
},
{
stmt: `SHOW TAG KEYS FROM mydb.myrp1./c.*/`,
s: `SHOW TAG KEYS WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM mydb.myrp1./c.*/`,
s: `SHOW TAG KEYS ON db0 WHERE _name =~ /c.*/`,
},
{
stmt: `SHOW TAG KEYS FROM mydb.myrp1.cpu WHERE region = 'uswest'`,
s: `SHOW TAG KEYS WHERE (_name = 'cpu') AND (region = 'uswest')`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM mydb.myrp1.cpu WHERE region = 'uswest'`,
s: `SHOW TAG KEYS ON db0 WHERE (_name = 'cpu') AND (region = 'uswest')`,
},
{
stmt: `SHOW TAG KEYS WHERE time > 0`,
s: `SHOW TAG KEYS WHERE time > 0`,
},
{
stmt: `SHOW TAG KEYS ON db0 WHERE time > 0`,
s: `SHOW TAG KEYS ON db0 WHERE time > 0`,
},
{
stmt: `SHOW TAG KEYS FROM cpu WHERE time > 0`,
s: `SHOW TAG KEYS WHERE (_name = 'cpu') AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM cpu WHERE time > 0`,
s: `SHOW TAG KEYS ON db0 WHERE (_name = 'cpu') AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS FROM /c.*/ WHERE time > 0`,
s: `SHOW TAG KEYS WHERE (_name =~ /c.*/) AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM /c.*/ WHERE time > 0`,
s: `SHOW TAG KEYS ON db0 WHERE (_name =~ /c.*/) AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS FROM cpu WHERE region = 'uswest' AND time > 0`,
s: `SHOW TAG KEYS WHERE (_name = 'cpu') AND (region = 'uswest' AND time > 0)`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM cpu WHERE region = 'uswest' AND time > 0`,
s: `SHOW TAG KEYS ON db0 WHERE (_name = 'cpu') AND (region = 'uswest' AND time > 0)`,
},
{
stmt: `SHOW TAG KEYS FROM mydb.myrp1.cpu WHERE time > 0`,
s: `SHOW TAG KEYS WHERE (_name = 'cpu') AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM mydb.myrp1.cpu WHERE time > 0`,
s: `SHOW TAG KEYS ON db0 WHERE (_name = 'cpu') AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS FROM mydb.myrp1./c.*/ WHERE time > 0`,
s: `SHOW TAG KEYS WHERE (_name =~ /c.*/) AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM mydb.myrp1./c.*/ WHERE time > 0`,
s: `SHOW TAG KEYS ON db0 WHERE (_name =~ /c.*/) AND (time > 0)`,
},
{
stmt: `SHOW TAG KEYS FROM mydb.myrp1.cpu WHERE region = 'uswest' AND time > 0`,
s: `SHOW TAG KEYS WHERE (_name = 'cpu') AND (region = 'uswest' AND time > 0)`,
},
{
stmt: `SHOW TAG KEYS ON db0 FROM mydb.myrp1.cpu WHERE region = 'uswest' AND time > 0`,
s: `SHOW TAG KEYS ON db0 WHERE (_name = 'cpu') AND (region = 'uswest' AND time > 0)`,
},
{
stmt: `SHOW TAG VALUES WITH KEY = "region"`,
s: `SHOW TAG VALUES WITH KEY = region WHERE _tagKey = 'region'`,
},
{
stmt: `SHOW TAG VALUES WITH KEY = "region" WHERE "region" = 'uswest'`,
s: `SHOW TAG VALUES WITH KEY = region WHERE (region = 'uswest') AND (_tagKey = 'region')`,
},
{
stmt: `SHOW TAG VALUES WITH KEY IN ("region", "server") WHERE "platform" = 'cloud'`,
s: `SHOW TAG VALUES WITH KEY IN (region, server) WHERE (platform = 'cloud') AND (_tagKey = 'region' OR _tagKey = 'server')`,
},
{
stmt: `SHOW TAG VALUES WITH KEY = "region" WHERE "region" = 'uswest' AND time > 0`,
s: `SHOW TAG VALUES WITH KEY = region WHERE (region = 'uswest' AND time > 0) AND (_tagKey = 'region')`,
},
{
stmt: `SHOW TAG VALUES WITH KEY = "region" ON db0`,
s: `SHOW TAG VALUES WITH KEY = region WHERE _tagKey = 'region'`,
},
{
stmt: `SHOW TAG VALUES FROM cpu WITH KEY = "region"`,
s: `SHOW TAG VALUES WITH KEY = region WHERE (_name = 'cpu') AND (_tagKey = 'region')`,
},
{
stmt: `SHOW TAG VALUES WITH KEY != "region"`,
s: `SHOW TAG VALUES WITH KEY != region WHERE _tagKey != 'region'`,
},
{
stmt: `SHOW TAG VALUES WITH KEY =~ /re.*/`,
s: `SHOW TAG VALUES WITH KEY =~ /re.*/ WHERE _tagKey =~ /re.*/`,
},
{
stmt: `SHOW TAG VALUES WITH KEY =~ /re.*/ WHERE time > 0`,
s: `SHOW TAG VALUES WITH KEY =~ /re.*/ WHERE (time > 0) AND (_tagKey =~ /re.*/)`,
},
{
stmt: `SHOW TAG VALUES WITH KEY !~ /re.*/`,
s: `SHOW TAG VALUES WITH KEY !~ /re.*/ WHERE _tagKey !~ /re.*/`,
},
{
stmt: `SHOW TAG VALUES WITH KEY !~ /re.*/ LIMIT 1`,
s: `SHOW TAG VALUES WITH KEY !~ /re.*/ WHERE _tagKey !~ /re.*/ LIMIT 1`,
},
{
stmt: `SHOW TAG VALUES WITH KEY !~ /re.*/ OFFSET 2`,
s: `SHOW TAG VALUES WITH KEY !~ /re.*/ WHERE _tagKey !~ /re.*/ OFFSET 2`,
},
{
stmt: `SELECT value FROM cpu`,
s: `SELECT value FROM cpu`,
},
}
for _, test := range tests {
t.Run(test.stmt, func(t *testing.T) {
stmt, err := influxql.ParseStatement(test.stmt)
if err != nil {
t.Errorf("error parsing statement: %s", err)
} else {
stmt, err = query.RewriteStatement(stmt)
if err != nil {
t.Errorf("error rewriting statement: %s", err)
} else if s := stmt.String(); s != test.s {
t.Errorf("error rendering string. expected %s, actual: %s", test.s, s)
}
}
})
}
}

126
influxql/query/subquery.go Normal file
View File

@ -0,0 +1,126 @@
package query
import (
"context"
"github.com/influxdata/influxql"
)
type subqueryBuilder struct {
ic IteratorCreator
stmt *influxql.SelectStatement
}
// buildAuxIterator constructs an auxiliary Iterator from a subquery.
func (b *subqueryBuilder) buildAuxIterator(ctx context.Context, opt IteratorOptions) (Iterator, error) {
// Map the desired auxiliary fields from the substatement.
indexes := b.mapAuxFields(opt.Aux)
subOpt, err := newIteratorOptionsSubstatement(ctx, b.stmt, opt)
if err != nil {
return nil, err
}
cur, err := buildCursor(ctx, b.stmt, b.ic, subOpt)
if err != nil {
return nil, err
}
// Filter the cursor by a condition if one was given.
if opt.Condition != nil {
cur = newFilterCursor(cur, opt.Condition)
}
// Construct the iterators for the subquery.
itr := NewIteratorMapper(cur, nil, indexes, subOpt)
if len(opt.GetDimensions()) != len(subOpt.GetDimensions()) {
itr = NewTagSubsetIterator(itr, opt)
}
return itr, nil
}
func (b *subqueryBuilder) mapAuxFields(auxFields []influxql.VarRef) []IteratorMap {
indexes := make([]IteratorMap, len(auxFields))
for i, name := range auxFields {
m := b.mapAuxField(&name)
if m == nil {
// If this field doesn't map to anything, use the NullMap so it
// shows up as null.
m = NullMap{}
}
indexes[i] = m
}
return indexes
}
func (b *subqueryBuilder) mapAuxField(name *influxql.VarRef) IteratorMap {
offset := 0
for i, f := range b.stmt.Fields {
if f.Name() == name.Val {
return FieldMap{
Index: i + offset,
// Cast the result of the field into the desired type.
Type: name.Type,
}
} else if call, ok := f.Expr.(*influxql.Call); ok && (call.Name == "top" || call.Name == "bottom") {
// We may match one of the arguments in "top" or "bottom".
if len(call.Args) > 2 {
for j, arg := range call.Args[1 : len(call.Args)-1] {
if arg, ok := arg.(*influxql.VarRef); ok && arg.Val == name.Val {
return FieldMap{
Index: i + j + 1,
Type: influxql.String,
}
}
}
// Increment the offset so we have the correct index for later fields.
offset += len(call.Args) - 2
}
}
}
// Unable to find this in the list of fields.
// Look within the dimensions and create a field if we find it.
for _, d := range b.stmt.Dimensions {
if d, ok := d.Expr.(*influxql.VarRef); ok && name.Val == d.Val {
return TagMap(d.Val)
}
}
// Unable to find any matches.
return nil
}
func (b *subqueryBuilder) buildVarRefIterator(ctx context.Context, expr *influxql.VarRef, opt IteratorOptions) (Iterator, error) {
// Look for the field or tag that is driving this query.
driver := b.mapAuxField(expr)
if driver == nil {
// Exit immediately if there is no driver. If there is no driver, there
// are no results. Period.
return nil, nil
}
// Map the auxiliary fields to their index in the subquery.
indexes := b.mapAuxFields(opt.Aux)
subOpt, err := newIteratorOptionsSubstatement(ctx, b.stmt, opt)
if err != nil {
return nil, err
}
cur, err := buildCursor(ctx, b.stmt, b.ic, subOpt)
if err != nil {
return nil, err
}
// Filter the cursor by a condition if one was given.
if opt.Condition != nil {
cur = newFilterCursor(cur, opt.Condition)
}
// Construct the iterators for the subquery.
itr := NewIteratorMapper(cur, driver, indexes, subOpt)
if len(opt.GetDimensions()) != len(subOpt.GetDimensions()) {
itr = NewTagSubsetIterator(itr, opt)
}
return itr, nil
}

461
influxql/query/subquery_test.go Normal file
View File

@ -0,0 +1,461 @@
package query_test
import (
"context"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/v1/models"
"github.com/influxdata/influxql"
)
type CreateIteratorFn func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator
func TestSubquery(t *testing.T) {
for _, test := range []struct {
Name string
Statement string
Fields map[string]influxql.DataType
MapShardsFn func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn
Rows []query.Row
}{
{
Name: "AuxiliaryFields",
Statement: `SELECT max / 2.0 FROM (SELECT max(value) FROM cpu GROUP BY time(5s)) WHERE time >= '1970-01-01T00:00:00Z' AND time < '1970-01-01T00:00:15Z'`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
if got, want := tr.MinTimeNano(), 0*Second; got != want {
t.Errorf("unexpected min time: got=%d want=%d", got, want)
}
if got, want := tr.MaxTimeNano(), 15*Second-1; got != want {
t.Errorf("unexpected max time: got=%d want=%d", got, want)
}
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
if got, want := opt.Expr.String(), "max(value::float)"; got != want {
t.Errorf("unexpected expression: got=%s want=%s", got, want)
}
return &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Time: 0 * Second, Value: 5},
{Name: "cpu", Time: 5 * Second, Value: 3},
{Name: "cpu", Time: 10 * Second, Value: 8},
}}
}
},
Rows: []query.Row{
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{2.5}},
{Time: 5 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{1.5}},
{Time: 10 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{float64(4)}},
},
},
{
Name: "AuxiliaryFields_WithWhereClause",
Statement: `SELECT host FROM (SELECT max(value), host FROM cpu GROUP BY time(5s)) WHERE max > 4 AND time >= '1970-01-01T00:00:00Z' AND time < '1970-01-01T00:00:15Z'`,
Fields: map[string]influxql.DataType{
"value": influxql.Float,
"host": influxql.Tag,
},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
if got, want := tr.MinTimeNano(), 0*Second; got != want {
t.Errorf("unexpected min time: got=%d want=%d", got, want)
}
if got, want := tr.MaxTimeNano(), 15*Second-1; got != want {
t.Errorf("unexpected max time: got=%d want=%d", got, want)
}
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
if got, want := opt.Expr.String(), "max(value::float)"; got != want {
t.Errorf("unexpected expression: got=%s want=%s", got, want)
}
if got, want := opt.Aux, []influxql.VarRef{{Val: "host", Type: influxql.Tag}}; !cmp.Equal(got, want) {
t.Errorf("unexpected auxiliary fields:\n%s", cmp.Diff(want, got))
}
return &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Time: 0 * Second, Value: 5, Aux: []interface{}{"server02"}},
{Name: "cpu", Time: 5 * Second, Value: 3, Aux: []interface{}{"server01"}},
{Name: "cpu", Time: 10 * Second, Value: 8, Aux: []interface{}{"server03"}},
}}
}
},
Rows: []query.Row{
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{"server02"}},
{Time: 10 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{"server03"}},
},
},
{
Name: "AuxiliaryFields_NonExistentField",
Statement: `SELECT host FROM (SELECT max(value) FROM cpu GROUP BY time(5s)) WHERE time >= '1970-01-01T00:00:00Z' AND time < '1970-01-01T00:00:15Z'`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
return &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Time: 0 * Second, Value: 5},
{Name: "cpu", Time: 5 * Second, Value: 3},
{Name: "cpu", Time: 10 * Second, Value: 8},
}}
}
},
Rows: []query.Row(nil),
},
{
Name: "AggregateOfMath",
Statement: `SELECT mean(percentage) FROM (SELECT value * 100.0 AS percentage FROM cpu) WHERE time >= '1970-01-01T00:00:00Z' AND time < '1970-01-01T00:00:15Z' GROUP BY time(5s)`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
if got, want := tr.MinTimeNano(), 0*Second; got != want {
t.Errorf("unexpected min time: got=%d want=%d", got, want)
}
if got, want := tr.MaxTimeNano(), 15*Second-1; got != want {
t.Errorf("unexpected max time: got=%d want=%d", got, want)
}
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
if got, want := opt.Expr, influxql.Expr(nil); got != want {
t.Errorf("unexpected expression: got=%s want=%s", got, want)
}
if got, want := opt.Aux, []influxql.VarRef{{Val: "value", Type: influxql.Float}}; !cmp.Equal(got, want) {
t.Errorf("unexpected auxiliary fields:\n%s", cmp.Diff(want, got))
}
return &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Time: 0 * Second, Aux: []interface{}{0.5}},
{Name: "cpu", Time: 2 * Second, Aux: []interface{}{1.0}},
{Name: "cpu", Time: 5 * Second, Aux: []interface{}{0.05}},
{Name: "cpu", Time: 8 * Second, Aux: []interface{}{0.45}},
{Name: "cpu", Time: 12 * Second, Aux: []interface{}{0.34}},
}}
}
},
Rows: []query.Row{
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{float64(75)}},
{Time: 5 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{float64(25)}},
{Time: 10 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{float64(34)}},
},
},
{
Name: "Cast",
Statement: `SELECT value::integer FROM (SELECT mean(value) AS value FROM cpu)`,
Fields: map[string]influxql.DataType{"value": influxql.Integer},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
if got, want := opt.Expr.String(), "mean(value::integer)"; got != want {
t.Errorf("unexpected expression: got=%s want=%s", got, want)
}
return &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Time: 0 * Second, Value: float64(20) / float64(6)},
}}
}
},
Rows: []query.Row{
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{int64(3)}},
},
},
{
Name: "CountTag",
Statement: `SELECT count(host) FROM (SELECT value, host FROM cpu) WHERE time >= '1970-01-01T00:00:00Z' AND time < '1970-01-01T00:00:15Z'`,
Fields: map[string]influxql.DataType{
"value": influxql.Float,
"host": influxql.Tag,
},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
if got, want := tr.MinTimeNano(), 0*Second; got != want {
t.Errorf("unexpected min time: got=%d want=%d", got, want)
}
if got, want := tr.MaxTimeNano(), 15*Second-1; got != want {
t.Errorf("unexpected max time: got=%d want=%d", got, want)
}
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
if got, want := opt.Aux, []influxql.VarRef{
{Val: "host", Type: influxql.Tag},
{Val: "value", Type: influxql.Float},
}; !cmp.Equal(got, want) {
t.Errorf("unexpected auxiliary fields:\n%s", cmp.Diff(want, got))
}
return &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Aux: []interface{}{"server01", 5.0}},
{Name: "cpu", Aux: []interface{}{"server02", 3.0}},
{Name: "cpu", Aux: []interface{}{"server03", 8.0}},
}}
}
},
Rows: []query.Row{
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{int64(3)}},
},
},
{
Name: "StripTags",
Statement: `SELECT max FROM (SELECT max(value) FROM cpu GROUP BY host) WHERE time >= '1970-01-01T00:00:00Z' AND time < '1970-01-01T00:00:15Z'`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
if got, want := tr.MinTimeNano(), 0*Second; got != want {
t.Errorf("unexpected min time: got=%d want=%d", got, want)
}
if got, want := tr.MaxTimeNano(), 15*Second-1; got != want {
t.Errorf("unexpected max time: got=%d want=%d", got, want)
}
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
if got, want := opt.Expr.String(), "max(value::float)"; got != want {
t.Errorf("unexpected expression: got=%s want=%s", got, want)
}
return &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Tags: ParseTags("host=server01"), Value: 5},
{Name: "cpu", Tags: ParseTags("host=server02"), Value: 3},
{Name: "cpu", Tags: ParseTags("host=server03"), Value: 8},
}}
}
},
Rows: []query.Row{
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{5.0}},
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{3.0}},
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{8.0}},
},
},
{
Name: "DifferentDimensionsWithSelectors",
Statement: `SELECT sum("max_min") FROM (
SELECT max("value") - min("value") FROM cpu GROUP BY time(30s), host
) WHERE time >= '1970-01-01T00:00:00Z' AND time < '1970-01-01T00:01:00Z' GROUP BY time(30s)`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
if got, want := tr.MinTimeNano(), 0*Second; got != want {
t.Errorf("unexpected min time: got=%d want=%d", got, want)
}
if got, want := tr.MaxTimeNano(), 60*Second-1; got != want {
t.Errorf("unexpected max time: got=%d want=%d", got, want)
}
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
var itr query.Iterator = &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 2},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: 7},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 20 * Second, Value: 3},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 0 * Second, Value: 8},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 10 * Second, Value: 3},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 20 * Second, Value: 7},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 30 * Second, Value: 2},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 40 * Second, Value: 1},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 50 * Second, Value: 9},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 30 * Second, Value: 2},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 40 * Second, Value: 2},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 50 * Second, Value: 2},
}}
if _, ok := opt.Expr.(*influxql.Call); ok {
i, err := query.NewCallIterator(itr, opt)
if err != nil {
panic(err)
}
itr = i
}
return itr
}
},
Rows: []query.Row{
{Time: 0 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{float64(10)}},
{Time: 30 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{float64(8)}},
},
},
{
Name: "TimeOrderingInTheOuterQuery",
Statement: `select * from (select last(value) from cpu group by host) order by time asc`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "cpu"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
var itr query.Iterator = &FloatIterator{Points: []query.FloatPoint{
{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 2},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: 7},
{Name: "cpu", Tags: ParseTags("host=A"), Time: 20 * Second, Value: 3},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 0 * Second, Value: 8},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 10 * Second, Value: 3},
{Name: "cpu", Tags: ParseTags("host=B"), Time: 19 * Second, Value: 7},
}}
if _, ok := opt.Expr.(*influxql.Call); ok {
i, err := query.NewCallIterator(itr, opt)
if err != nil {
panic(err)
}
itr = i
}
return itr
}
},
Rows: []query.Row{
{Time: 19 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{"B", float64(7)}},
{Time: 20 * Second, Series: query.Series{Name: "cpu"}, Values: []interface{}{"A", float64(3)}},
},
},
{
Name: "TimeZone",
Statement: `SELECT * FROM (SELECT * FROM cpu WHERE time >= '2019-04-17 09:00:00' and time < '2019-04-17 10:00:00' TZ('America/Chicago'))`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := time.Unix(0, opt.StartTime).UTC(), mustParseTime("2019-04-17T14:00:00Z"); !got.Equal(want) {
t.Errorf("unexpected min time: got=%q want=%q", got, want)
}
if got, want := time.Unix(0, opt.EndTime).UTC(), mustParseTime("2019-04-17T15:00:00Z").Add(-1); !got.Equal(want) {
t.Errorf("unexpected max time: got=%q want=%q", got, want)
}
return &FloatIterator{}
}
},
},
{
Name: "DifferentDimensionsOrderByDesc",
Statement: `SELECT value, mytag FROM (SELECT last(value) AS value FROM testing GROUP BY mytag) ORDER BY desc`,
Fields: map[string]influxql.DataType{"value": influxql.Float},
MapShardsFn: func(t *testing.T, tr influxql.TimeRange) CreateIteratorFn {
return func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) query.Iterator {
if got, want := m.Name, "testing"; got != want {
t.Errorf("unexpected source: got=%s want=%s", got, want)
}
if opt.Ascending {
t.Error("expected iterator to be descending, not ascending")
}
var itr query.Iterator = &FloatIterator{Points: []query.FloatPoint{
{Name: "testing", Tags: ParseTags("mytag=c"), Time: mustParseTime("2019-06-25T22:36:20.93605779Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=c"), Time: mustParseTime("2019-06-25T22:36:20.671604877Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=c"), Time: mustParseTime("2019-06-25T22:36:20.255794481Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=b"), Time: mustParseTime("2019-06-25T22:36:18.176662543Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=b"), Time: mustParseTime("2019-06-25T22:36:17.815979113Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=b"), Time: mustParseTime("2019-06-25T22:36:17.265031598Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=a"), Time: mustParseTime("2019-06-25T22:36:15.144253616Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=a"), Time: mustParseTime("2019-06-25T22:36:14.719167205Z").UnixNano(), Value: 2},
{Name: "testing", Tags: ParseTags("mytag=a"), Time: mustParseTime("2019-06-25T22:36:13.711721316Z").UnixNano(), Value: 2},
}}
if _, ok := opt.Expr.(*influxql.Call); ok {
i, err := query.NewCallIterator(itr, opt)
if err != nil {
panic(err)
}
itr = i
}
return itr
}
},
Rows: []query.Row{
{Time: mustParseTime("2019-06-25T22:36:20.93605779Z").UnixNano(), Series: query.Series{Name: "testing"}, Values: []interface{}{float64(2), "c"}},
{Time: mustParseTime("2019-06-25T22:36:18.176662543Z").UnixNano(), Series: query.Series{Name: "testing"}, Values: []interface{}{float64(2), "b"}},
{Time: mustParseTime("2019-06-25T22:36:15.144253616Z").UnixNano(), Series: query.Series{Name: "testing"}, Values: []interface{}{float64(2), "a"}},
},
},
} {
t.Run(test.Name, func(t *testing.T) {
shardMapper := ShardMapper{
MapShardsFn: func(sources influxql.Sources, tr influxql.TimeRange) query.ShardGroup {
fn := test.MapShardsFn(t, tr)
return &ShardGroup{
Fields: test.Fields,
CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) {
return fn(ctx, m, opt), nil
},
}
},
}
stmt := MustParseSelectStatement(test.Statement)
stmt.OmitTime = true
cur, err := query.Select(context.Background(), stmt, &shardMapper, query.SelectOptions{})
if err != nil {
t.Fatalf("unexpected parse error: %s", err)
} else if a, err := ReadCursor(cur); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if diff := cmp.Diff(test.Rows, a); diff != "" {
t.Fatalf("unexpected points:\n%s", diff)
}
})
}
}
type openAuthorizer struct{}
func (*openAuthorizer) AuthorizeDatabase(p influxql.Privilege, name string) bool { return true }
func (*openAuthorizer) AuthorizeQuery(database string, query *influxql.Query) error { return nil }
func (*openAuthorizer) AuthorizeSeriesRead(database string, measurement []byte, tags models.Tags) bool {
return true
}
func (*openAuthorizer) AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool {
return true
}
// Ensure that the subquery gets passed the query authorizer.
func TestSubquery_Authorizer(t *testing.T) {
auth := &openAuthorizer{}
shardMapper := ShardMapper{
MapShardsFn: func(sources influxql.Sources, tr influxql.TimeRange) query.ShardGroup {
return &ShardGroup{
Fields: map[string]influxql.DataType{
"value": influxql.Float,
},
CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) {
if opt.Authorizer != auth {
t.Errorf("query authorizer has not been set")
}
return nil, nil
},
}
},
}
stmt := MustParseSelectStatement(`SELECT max(value) FROM (SELECT value FROM cpu)`)
cur, err := query.Select(context.Background(), stmt, &shardMapper, query.SelectOptions{
Authorizer: auth,
})
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
cur.Close()
}
// Ensure that the subquery gets passed the max series limit.
func TestSubquery_MaxSeriesN(t *testing.T) {
shardMapper := ShardMapper{
MapShardsFn: func(sources influxql.Sources, tr influxql.TimeRange) query.ShardGroup {
return &ShardGroup{
Fields: map[string]influxql.DataType{
"value": influxql.Float,
},
CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) {
if opt.MaxSeriesN != 1000 {
t.Errorf("max series limit has not been set")
}
return nil, nil
},
}
},
}
stmt := MustParseSelectStatement(`SELECT max(value) FROM (SELECT value FROM cpu)`)
cur, err := query.Select(context.Background(), stmt, &shardMapper, query.SelectOptions{
MaxSeriesN: 1000,
})
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
cur.Close()
}

319
influxql/query/task_manager.go Normal file
View File

@ -0,0 +1,319 @@
package query
import (
"bytes"
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/influxdata/influxdb/v2/v1/models"
"github.com/influxdata/influxql"
"go.uber.org/zap"
)
const (
// DefaultQueryTimeout is the default timeout for executing a query.
// A value of zero will have no query timeout.
DefaultQueryTimeout = time.Duration(0)
)
type TaskStatus int
const (
// RunningTask is set when the task is running.
RunningTask TaskStatus = iota + 1
// KilledTask is set when the task is killed, but resources are still
// being used.
KilledTask
)
func (t TaskStatus) String() string {
switch t {
case RunningTask:
return "running"
case KilledTask:
return "killed"
default:
return "unknown"
}
}
func (t TaskStatus) MarshalJSON() ([]byte, error) {
s := t.String()
return json.Marshal(s)
}
func (t *TaskStatus) UnmarshalJSON(data []byte) error {
if bytes.Equal(data, []byte("running")) {
*t = RunningTask
} else if bytes.Equal(data, []byte("killed")) {
*t = KilledTask
} else if bytes.Equal(data, []byte("unknown")) {
*t = TaskStatus(0)
} else {
return fmt.Errorf("unknown task status: %s", string(data))
}
return nil
}
// TaskManager takes care of all aspects related to managing running queries.
type TaskManager struct {
// Query execution timeout.
QueryTimeout time.Duration
// Log queries if they are slower than this time.
// If zero, slow queries will never be logged.
LogQueriesAfter time.Duration
// Maximum number of concurrent queries.
MaxConcurrentQueries int
// Logger to use for all logging.
// Defaults to discarding all log output.
Logger *zap.Logger
// Used for managing and tracking running queries.
queries map[uint64]*Task
nextID uint64
mu sync.RWMutex
shutdown bool
}
// NewTaskManager creates a new TaskManager.
func NewTaskManager() *TaskManager {
return &TaskManager{
QueryTimeout: DefaultQueryTimeout,
Logger: zap.NewNop(),
queries: make(map[uint64]*Task),
nextID: 1,
}
}
// ExecuteStatement executes a statement containing one of the task management queries.
func (t *TaskManager) ExecuteStatement(stmt influxql.Statement, ctx *ExecutionContext) error {
switch stmt := stmt.(type) {
case *influxql.ShowQueriesStatement:
rows, err := t.executeShowQueriesStatement(stmt)
if err != nil {
return err
}
ctx.Send(&Result{
Series: rows,
})
case *influxql.KillQueryStatement:
var messages []*Message
if ctx.ReadOnly {
messages = append(messages, ReadOnlyWarning(stmt.String()))
}
if err := t.executeKillQueryStatement(stmt); err != nil {
return err
}
ctx.Send(&Result{
Messages: messages,
})
default:
return ErrInvalidQuery
}
return nil
}
func (t *TaskManager) executeKillQueryStatement(stmt *influxql.KillQueryStatement) error {
return t.KillQuery(stmt.QueryID)
}
func (t *TaskManager) executeShowQueriesStatement(q *influxql.ShowQueriesStatement) (models.Rows, error) {
t.mu.RLock()
defer t.mu.RUnlock()
now := time.Now()
values := make([][]interface{}, 0, len(t.queries))
for id, qi := range t.queries {
d := now.Sub(qi.startTime)
switch {
case d >= time.Second:
d = d - (d % time.Second)
case d >= time.Millisecond:
d = d - (d % time.Millisecond)
case d >= time.Microsecond:
d = d - (d % time.Microsecond)
}
values = append(values, []interface{}{id, qi.query, qi.database, d.String(), qi.status.String()})
}
return []*models.Row{{
Columns: []string{"qid", "query", "database", "duration", "status"},
Values: values,
}}, nil
}
func (t *TaskManager) queryError(qid uint64, err error) {
t.mu.RLock()
query := t.queries[qid]
t.mu.RUnlock()
if query != nil {
query.setError(err)
}
}
// AttachQuery attaches a running query to be managed by the TaskManager.
// Returns the query id of the newly attached query or an error if it was
// unable to assign a query id or attach the query to the TaskManager.
// This function also returns a channel that will be closed when this
// query finishes running.
//
// After a query finishes running, the system is free to reuse a query id.
func (t *TaskManager) AttachQuery(q *influxql.Query, opt ExecutionOptions, interrupt <-chan struct{}) (*ExecutionContext, func(), error) {
t.mu.Lock()
defer t.mu.Unlock()
if t.shutdown {
return nil, nil, ErrQueryEngineShutdown
}
if t.MaxConcurrentQueries > 0 && len(t.queries) >= t.MaxConcurrentQueries {
return nil, nil, ErrMaxConcurrentQueriesLimitExceeded(len(t.queries), t.MaxConcurrentQueries)
}
qid := t.nextID
query := &Task{
query: q.String(),
database: opt.Database,
status: RunningTask,
startTime: time.Now(),
closing: make(chan struct{}),
monitorCh: make(chan error),
}
t.queries[qid] = query
go t.waitForQuery(qid, query.closing, interrupt, query.monitorCh)
if t.LogQueriesAfter != 0 {
go query.monitor(func(closing <-chan struct{}) error {
timer := time.NewTimer(t.LogQueriesAfter)
defer timer.Stop()
select {
case <-timer.C:
t.Logger.Warn(fmt.Sprintf("Detected slow query: %s (qid: %d, database: %s, threshold: %s)",
query.query, qid, query.database, t.LogQueriesAfter))
case <-closing:
}
return nil
})
}
t.nextID++
ctx := &ExecutionContext{
Context: context.Background(),
QueryID: qid,
task: query,
ExecutionOptions: opt,
}
ctx.watch()
return ctx, func() { t.DetachQuery(qid) }, nil
}
// KillQuery enters a query into the killed state and closes the channel
// from the TaskManager. This method can be used to forcefully terminate a
// running query.
func (t *TaskManager) KillQuery(qid uint64) error {
t.mu.Lock()
query := t.queries[qid]
t.mu.Unlock()
if query == nil {
return fmt.Errorf("no such query id: %d", qid)
}
return query.kill()
}
// DetachQuery removes a query from the query table. If the query is not in the
// killed state, this will also close the related channel.
func (t *TaskManager) DetachQuery(qid uint64) error {
t.mu.Lock()
defer t.mu.Unlock()
query := t.queries[qid]
if query == nil {
return fmt.Errorf("no such query id: %d", qid)
}
query.close()
delete(t.queries, qid)
return nil
}
// QueryInfo represents the information for a query.
type QueryInfo struct {
ID uint64 `json:"id"`
Query string `json:"query"`
Database string `json:"database"`
Duration time.Duration `json:"duration"`
Status TaskStatus `json:"status"`
}
// Queries returns a list of all running queries with information about them.
func (t *TaskManager) Queries() []QueryInfo {
t.mu.RLock()
defer t.mu.RUnlock()
now := time.Now()
queries := make([]QueryInfo, 0, len(t.queries))
for id, qi := range t.queries {
queries = append(queries, QueryInfo{
ID: id,
Query: qi.query,
Database: qi.database,
Duration: now.Sub(qi.startTime),
Status: qi.status,
})
}
return queries
}
func (t *TaskManager) waitForQuery(qid uint64, interrupt <-chan struct{}, closing <-chan struct{}, monitorCh <-chan error) {
var timerCh <-chan time.Time
if t.QueryTimeout != 0 {
timer := time.NewTimer(t.QueryTimeout)
timerCh = timer.C
defer timer.Stop()
}
select {
case <-closing:
t.queryError(qid, ErrQueryInterrupted)
case err := <-monitorCh:
if err == nil {
break
}
t.queryError(qid, err)
case <-timerCh:
t.queryError(qid, ErrQueryTimeoutLimitExceeded)
case <-interrupt:
// Query was manually closed so exit the select.
return
}
t.KillQuery(qid)
}
// Close kills all running queries and prevents new queries from being attached.
func (t *TaskManager) Close() error {
t.mu.Lock()
defer t.mu.Unlock()
t.shutdown = true
for _, query := range t.queries {
query.setError(ErrQueryEngineShutdown)
query.close()
}
t.queries = nil
return nil
}

37
influxql/query/tmpldata Normal file
View File

@ -0,0 +1,37 @@
[
{
"Name":"Float",
"name":"float",
"Type":"float64",
"Nil":"0",
"Zero":"float64(0)"
},
{
"Name":"Integer",
"name":"integer",
"Type":"int64",
"Nil":"0",
"Zero":"int64(0)"
},
{
"Name":"Unsigned",
"name":"unsigned",
"Type":"uint64",
"Nil":"0",
"Zero":"uint64(0)"
},
{
"Name":"String",
"name":"string",
"Type":"string",
"Nil":"\"\"",
"Zero":"\"\""
},
{
"Name":"Boolean",
"name":"boolean",
"Type":"bool",
"Nil":"false",
"Zero":"false"
}
]

185
pkg/deep/equal.go Normal file
View File

@ -0,0 +1,185 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// License.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Package deep provides a deep equality check for use in tests.
package deep // import "github.com/influxdata/influxdb/v2/pkg/deep"
import (
"fmt"
"math"
"reflect"
)
// Equal is a copy of reflect.DeepEqual except that it treats NaN == NaN as true.
func Equal(a1, a2 interface{}) bool {
if a1 == nil || a2 == nil {
return a1 == a2
}
v1 := reflect.ValueOf(a1)
v2 := reflect.ValueOf(a2)
if v1.Type() != v2.Type() {
return false
}
return deepValueEqual(v1, v2, make(map[visit]bool), 0)
}
// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func deepValueEqual(v1, v2 reflect.Value, visited map[visit]bool, depth int) bool {
if !v1.IsValid() || !v2.IsValid() {
return v1.IsValid() == v2.IsValid()
}
if v1.Type() != v2.Type() {
return false
}
// if depth > 10 { panic("deepValueEqual") } // for debugging
hard := func(k reflect.Kind) bool {
switch k {
case reflect.Array, reflect.Map, reflect.Slice, reflect.Struct:
return true
}
return false
}
if v1.CanAddr() && v2.CanAddr() && hard(v1.Kind()) {
addr1 := v1.UnsafeAddr()
addr2 := v2.UnsafeAddr()
if addr1 > addr2 {
// Canonicalize order to reduce number of entries in visited.
addr1, addr2 = addr2, addr1
}
// Short circuit if references are identical ...
if addr1 == addr2 {
return true
}
// ... or already seen
typ := v1.Type()
v := visit{addr1, addr2, typ}
if visited[v] {
return true
}
// Remember for later.
visited[v] = true
}
switch v1.Kind() {
case reflect.Array:
for i := 0; i < v1.Len(); i++ {
if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
return false
}
}
return true
case reflect.Slice:
if v1.IsNil() != v2.IsNil() {
return false
}
if v1.Len() != v2.Len() {
return false
}
if v1.Pointer() == v2.Pointer() {
return true
}
for i := 0; i < v1.Len(); i++ {
if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
return false
}
}
return true
case reflect.Interface:
if v1.IsNil() || v2.IsNil() {
return v1.IsNil() == v2.IsNil()
}
return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
case reflect.Ptr:
return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
case reflect.Struct:
for i, n := 0, v1.NumField(); i < n; i++ {
if !deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) {
return false
}
}
return true
case reflect.Map:
if v1.IsNil() != v2.IsNil() {
return false
}
if v1.Len() != v2.Len() {
return false
}
if v1.Pointer() == v2.Pointer() {
return true
}
for _, k := range v1.MapKeys() {
if !deepValueEqual(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) {
return false
}
}
return true
case reflect.Func:
if v1.IsNil() && v2.IsNil() {
return true
}
// Can't do better than this:
return false
case reflect.String:
return v1.String() == v2.String()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v1.Int() == v2.Int()
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return v1.Uint() == v2.Uint()
case reflect.Float32, reflect.Float64:
// Special handling for floats so that NaN == NaN is true.
f1, f2 := v1.Float(), v2.Float()
if math.IsNaN(f1) && math.IsNaN(f2) {
return true
}
return f1 == f2
case reflect.Bool:
return v1.Bool() == v2.Bool()
default:
panic(fmt.Sprintf("cannot compare type: %s", v1.Kind().String()))
}
}
// During deepValueEqual, must keep track of checks that are
// in progress. The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited comparisons are stored in a map indexed by visit.
type visit struct {
a1 uintptr
a2 uintptr
typ reflect.Type
}

2
pkg/escape/bytes.go
View File

@ -1,6 +1,6 @@
// Package escape contains utilities for escaping parts of InfluxQL
// and InfluxDB line protocol.
package escape // import "github.com/influxdata/influxdb/pkg/escape"
package escape // import "github.com/influxdata/influxdb/v2/pkg/escape"
import (
"bytes"

173
pkg/estimator/hll/compressed.go Normal file
View File

@ -0,0 +1,173 @@
package hll
import "encoding/binary"
// Original author of this file is github.com/clarkduvall/hyperloglog
type iterable interface {
decode(i int, last uint32) (uint32, int)
Len() int
Iter() *iterator
}
type iterator struct {
i int
last uint32
v iterable
}
func (iter *iterator) Next() uint32 {
n, i := iter.v.decode(iter.i, iter.last)
iter.last = n
iter.i = i
return n
}
func (iter *iterator) Peek() uint32 {
n, _ := iter.v.decode(iter.i, iter.last)
return n
}
func (iter iterator) HasNext() bool {
return iter.i < iter.v.Len()
}
type compressedList struct {
count uint32
last uint32
b variableLengthList
}
func (v *compressedList) Clone() *compressedList {
if v == nil {
return nil
}
newV := &compressedList{
count: v.count,
last: v.last,
}
newV.b = make(variableLengthList, len(v.b))
copy(newV.b, v.b)
return newV
}
func (v *compressedList) MarshalBinary() (data []byte, err error) {
// Marshal the variableLengthList
bdata, err := v.b.MarshalBinary()
if err != nil {
return nil, err
}
// At least 4 bytes for the two fixed sized values plus the size of bdata.
data = make([]byte, 0, 4+4+len(bdata))
// Marshal the count and last values.
data = append(data, []byte{
// Number of items in the list.
byte(v.count >> 24),
byte(v.count >> 16),
byte(v.count >> 8),
byte(v.count),
// The last item in the list.
byte(v.last >> 24),
byte(v.last >> 16),
byte(v.last >> 8),
byte(v.last),
}...)
// Append the list
return append(data, bdata...), nil
}
func (v *compressedList) UnmarshalBinary(data []byte) error {
// Set the count.
v.count, data = binary.BigEndian.Uint32(data[:4]), data[4:]
// Set the last value.
v.last, data = binary.BigEndian.Uint32(data[:4]), data[4:]
// Set the list.
sz, data := binary.BigEndian.Uint32(data[:4]), data[4:]
v.b = make([]uint8, sz)
for i := uint32(0); i < sz; i++ {
v.b[i] = uint8(data[i])
}
return nil
}
func newCompressedList(size int) *compressedList {
v := &compressedList{}
v.b = make(variableLengthList, 0, size)
return v
}
func (v *compressedList) Len() int {
return len(v.b)
}
func (v *compressedList) decode(i int, last uint32) (uint32, int) {
n, i := v.b.decode(i, last)
return n + last, i
}
func (v *compressedList) Append(x uint32) {
v.count++
v.b = v.b.Append(x - v.last)
v.last = x
}
func (v *compressedList) Iter() *iterator {
return &iterator{0, 0, v}
}
type variableLengthList []uint8
func (v variableLengthList) MarshalBinary() (data []byte, err error) {
// 4 bytes for the size of the list, and a byte for each element in the
// list.
data = make([]byte, 0, 4+v.Len())
// Length of the list. We only need 32 bits because the size of the set
// couldn't exceed that on 32 bit architectures.
sz := v.Len()
data = append(data, []byte{
byte(sz >> 24),
byte(sz >> 16),
byte(sz >> 8),
byte(sz),
}...)
// Marshal each element in the list.
for i := 0; i < sz; i++ {
data = append(data, byte(v[i]))
}
return data, nil
}
func (v variableLengthList) Len() int {
return len(v)
}
func (v *variableLengthList) Iter() *iterator {
return &iterator{0, 0, v}
}
func (v variableLengthList) decode(i int, last uint32) (uint32, int) {
var x uint32
j := i
for ; v[j]&0x80 != 0; j++ {
x |= uint32(v[j]&0x7f) << (uint(j-i) * 7)
}
x |= uint32(v[j]) << (uint(j-i) * 7)
return x, j + 1
}
func (v variableLengthList) Append(x uint32) variableLengthList {
for x&0xffffff80 != 0 {
v = append(v, uint8((x&0x7f)|0x80))
x >>= 7
}
return append(v, uint8(x&0x7f))
}

495
pkg/estimator/hll/hll.go Normal file
View File

@ -0,0 +1,495 @@
// Package hll contains a HyperLogLog++ with a LogLog-Beta bias correction implementation that is adapted (mostly
// copied) from an implementation provided by Clark DuVall
// github.com/clarkduvall/hyperloglog.
//
// The differences are that the implementation in this package:
//
// * uses an AMD64 optimised xxhash algorithm instead of murmur;
// * uses some AMD64 optimisations for things like clz;
// * works with []byte rather than a Hash64 interface, to reduce allocations;
// * implements encoding.BinaryMarshaler and encoding.BinaryUnmarshaler
//
// Based on some rough benchmarking, this implementation of HyperLogLog++ is
// around twice as fast as the github.com/clarkduvall/hyperloglog implementation.
package hll
import (
"encoding/binary"
"errors"
"fmt"
"math"
"math/bits"
"sort"
"unsafe"
"github.com/cespare/xxhash"
"github.com/influxdata/influxdb/v2/pkg/estimator"
)
// Current version of HLL implementation.
const version uint8 = 2
// DefaultPrecision is the default precision.
const DefaultPrecision = 16
func beta(ez float64) float64 {
zl := math.Log(ez + 1)
return -0.37331876643753059*ez +
-1.41704077448122989*zl +
0.40729184796612533*math.Pow(zl, 2) +
1.56152033906584164*math.Pow(zl, 3) +
-0.99242233534286128*math.Pow(zl, 4) +
0.26064681399483092*math.Pow(zl, 5) +
-0.03053811369682807*math.Pow(zl, 6) +
0.00155770210179105*math.Pow(zl, 7)
}
// Plus implements the Hyperloglog++ algorithm, described in the following
// paper: http://static.googleusercontent.com/media/research.google.com/en//pubs/archive/40671.pdf
//
// The HyperLogLog++ algorithm provides cardinality estimations.
type Plus struct {
// hash function used to hash values to add to the sketch.
hash func([]byte) uint64
p uint8 // precision.
pp uint8 // p' (sparse) precision to be used when p ∈ [4..pp] and pp < 64.
m uint32 // Number of substream used for stochastic averaging of stream.
mp uint32 // m' (sparse) number of substreams.
alpha float64 // alpha is used for bias correction.
sparse bool // Should we use a sparse sketch representation.
tmpSet set
denseList []uint8 // The dense representation of the HLL.
sparseList *compressedList // values that can be stored in the sparse representation.
}
// NewPlus returns a new Plus with precision p. p must be between 4 and 18.
func NewPlus(p uint8) (*Plus, error) {
if p > 18 || p < 4 {
return nil, errors.New("precision must be between 4 and 18")
}
// p' = 25 is used in the Google paper.
pp := uint8(25)
hll := &Plus{
hash: xxhash.Sum64,
p: p,
pp: pp,
m: 1 << p,
mp: 1 << pp,
tmpSet: set{},
sparse: true,
}
hll.sparseList = newCompressedList(int(hll.m))
// Determine alpha.
switch hll.m {
case 16:
hll.alpha = 0.673
case 32:
hll.alpha = 0.697
case 64:
hll.alpha = 0.709
default:
hll.alpha = 0.7213 / (1 + 1.079/float64(hll.m))
}
return hll, nil
}
// Bytes estimates the memory footprint of this Plus, in bytes.
func (h *Plus) Bytes() int {
var b int
b += len(h.tmpSet) * 4
b += cap(h.denseList)
if h.sparseList != nil {
b += int(unsafe.Sizeof(*h.sparseList))
b += cap(h.sparseList.b)
}
b += int(unsafe.Sizeof(*h))
return b
}
// NewDefaultPlus creates a new Plus with the default precision.
func NewDefaultPlus() *Plus {
p, err := NewPlus(DefaultPrecision)
if err != nil {
panic(err)
}
return p
}
// Clone returns a deep copy of h.
func (h *Plus) Clone() estimator.Sketch {
var hll = &Plus{
hash: h.hash,
p: h.p,
pp: h.pp,
m: h.m,
mp: h.mp,
alpha: h.alpha,
sparse: h.sparse,
tmpSet: h.tmpSet.Clone(),
sparseList: h.sparseList.Clone(),
}
hll.denseList = make([]uint8, len(h.denseList))
copy(hll.denseList, h.denseList)
return hll
}
// Add adds a new value to the HLL.
func (h *Plus) Add(v []byte) {
x := h.hash(v)
if h.sparse {
h.tmpSet.add(h.encodeHash(x))
if uint32(len(h.tmpSet))*100 > h.m {
h.mergeSparse()
if uint32(h.sparseList.Len()) > h.m {
h.toNormal()
}
}
} else {
i := bextr(x, 64-h.p, h.p) // {x63,...,x64-p}
w := x<<h.p | 1<<(h.p-1) // {x63-p,...,x0}
rho := uint8(bits.LeadingZeros64(w)) + 1
if rho > h.denseList[i] {
h.denseList[i] = rho
}
}
}
// Count returns a cardinality estimate.
func (h *Plus) Count() uint64 {
if h == nil {
return 0 // Nothing to do.
}
if h.sparse {
h.mergeSparse()
return uint64(h.linearCount(h.mp, h.mp-uint32(h.sparseList.count)))
}
sum := 0.0
m := float64(h.m)
var count float64
for _, val := range h.denseList {
sum += 1.0 / float64(uint32(1)<<val)
if val == 0 {
count++
}
}
// Use LogLog-Beta bias estimation
return uint64((h.alpha * m * (m - count) / (beta(count) + sum)) + 0.5)
}
// Merge takes another HyperLogLogPlus and combines it with HyperLogLogPlus h.
// If HyperLogLogPlus h is using the sparse representation, it will be converted
// to the normal representation.
func (h *Plus) Merge(s estimator.Sketch) error {
if s == nil {
// Nothing to do
return nil
}
other, ok := s.(*Plus)
if !ok {
return fmt.Errorf("wrong type for merging: %T", other)
}
if h.p != other.p {
return errors.New("precisions must be equal")
}
if h.sparse {
h.toNormal()
}
if other.sparse {
for k := range other.tmpSet {
i, r := other.decodeHash(k)
if h.denseList[i] < r {
h.denseList[i] = r
}
}
for iter := other.sparseList.Iter(); iter.HasNext(); {
i, r := other.decodeHash(iter.Next())
if h.denseList[i] < r {
h.denseList[i] = r
}
}
} else {
for i, v := range other.denseList {
if v > h.denseList[i] {
h.denseList[i] = v
}
}
}
return nil
}
// MarshalBinary implements the encoding.BinaryMarshaler interface.
func (h *Plus) MarshalBinary() (data []byte, err error) {
if h == nil {
return nil, nil
}
// Marshal a version marker.
data = append(data, version)
// Marshal precision.
data = append(data, byte(h.p))
if h.sparse {
// It's using the sparse representation.
data = append(data, byte(1))
// Add the tmp_set
tsdata, err := h.tmpSet.MarshalBinary()
if err != nil {
return nil, err
}
data = append(data, tsdata...)
// Add the sparse representation
sdata, err := h.sparseList.MarshalBinary()
if err != nil {
return nil, err
}
return append(data, sdata...), nil
}
// It's using the dense representation.
data = append(data, byte(0))
// Add the dense sketch representation.
sz := len(h.denseList)
data = append(data, []byte{
byte(sz >> 24),
byte(sz >> 16),
byte(sz >> 8),
byte(sz),
}...)
// Marshal each element in the list.
for i := 0; i < len(h.denseList); i++ {
data = append(data, byte(h.denseList[i]))
}
return data, nil
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
func (h *Plus) UnmarshalBinary(data []byte) error {
if len(data) < 12 {
return fmt.Errorf("provided buffer %v too short for initializing HLL sketch", data)
}
// Unmarshal version. We may need this in the future if we make
// non-compatible changes.
_ = data[0]
// Unmarshal precision.
p := uint8(data[1])
newh, err := NewPlus(p)
if err != nil {
return err
}
*h = *newh
// h is now initialised with the correct precision. We just need to fill the
// rest of the details out.
if data[2] == byte(1) {
// Using the sparse representation.
h.sparse = true
// Unmarshal the tmp_set.
tssz := binary.BigEndian.Uint32(data[3:7])
h.tmpSet = make(map[uint32]struct{}, tssz)
// We need to unmarshal tssz values in total, and each value requires us
// to read 4 bytes.
tsLastByte := int((tssz * 4) + 7)
for i := 7; i < tsLastByte; i += 4 {
k := binary.BigEndian.Uint32(data[i : i+4])
h.tmpSet[k] = struct{}{}
}
// Unmarshal the sparse representation.
return h.sparseList.UnmarshalBinary(data[tsLastByte:])
}
// Using the dense representation.
h.sparse = false
dsz := int(binary.BigEndian.Uint32(data[3:7]))
h.denseList = make([]uint8, 0, dsz)
for i := 7; i < dsz+7; i++ {
h.denseList = append(h.denseList, uint8(data[i]))
}
return nil
}
func (h *Plus) mergeSparse() {
if len(h.tmpSet) == 0 {
return
}
keys := make(uint64Slice, 0, len(h.tmpSet))
for k := range h.tmpSet {
keys = append(keys, k)
}
sort.Sort(keys)
newList := newCompressedList(int(h.m))
for iter, i := h.sparseList.Iter(), 0; iter.HasNext() || i < len(keys); {
if !iter.HasNext() {
newList.Append(keys[i])
i++
continue
}
if i >= len(keys) {
newList.Append(iter.Next())
continue
}
x1, x2 := iter.Peek(), keys[i]
if x1 == x2 {
newList.Append(iter.Next())
i++
} else if x1 > x2 {
newList.Append(x2)
i++
} else {
newList.Append(iter.Next())
}
}
h.sparseList = newList
h.tmpSet = set{}
}
// Convert from sparse representation to dense representation.
func (h *Plus) toNormal() {
if len(h.tmpSet) > 0 {
h.mergeSparse()
}
h.denseList = make([]uint8, h.m)
for iter := h.sparseList.Iter(); iter.HasNext(); {
i, r := h.decodeHash(iter.Next())
if h.denseList[i] < r {
h.denseList[i] = r
}
}
h.sparse = false
h.tmpSet = nil
h.sparseList = nil
}
// Encode a hash to be used in the sparse representation.
func (h *Plus) encodeHash(x uint64) uint32 {
idx := uint32(bextr(x, 64-h.pp, h.pp))
if bextr(x, 64-h.pp, h.pp-h.p) == 0 {
zeros := bits.LeadingZeros64((bextr(x, 0, 64-h.pp)<<h.pp)|(1<<h.pp-1)) + 1
return idx<<7 | uint32(zeros<<1) | 1
}
return idx << 1
}
// Decode a hash from the sparse representation.
func (h *Plus) decodeHash(k uint32) (uint32, uint8) {
var r uint8
if k&1 == 1 {
r = uint8(bextr32(k, 1, 6)) + h.pp - h.p
} else {
r = uint8(bits.LeadingZeros32(k<<(32-h.pp+h.p-1)) + 1)
}
return h.getIndex(k), r
}
func (h *Plus) getIndex(k uint32) uint32 {
if k&1 == 1 {
return bextr32(k, 32-h.p, h.p)
}
return bextr32(k, h.pp-h.p+1, h.p)
}
func (h *Plus) linearCount(m uint32, v uint32) float64 {
fm := float64(m)
return fm * math.Log(fm/float64(v))
}
type uint64Slice []uint32
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
type set map[uint32]struct{}
func (s set) Clone() set {
if s == nil {
return nil
}
newS := make(map[uint32]struct{}, len(s))
for k, v := range s {
newS[k] = v
}
return newS
}
func (s set) MarshalBinary() (data []byte, err error) {
// 4 bytes for the size of the set, and 4 bytes for each key.
// list.
data = make([]byte, 0, 4+(4*len(s)))
// Length of the set. We only need 32 bits because the size of the set
// couldn't exceed that on 32 bit architectures.
sl := len(s)
data = append(data, []byte{
byte(sl >> 24),
byte(sl >> 16),
byte(sl >> 8),
byte(sl),
}...)
// Marshal each element in the set.
for k := range s {
data = append(data, []byte{
byte(k >> 24),
byte(k >> 16),
byte(k >> 8),
byte(k),
}...)
}
return data, nil
}
func (s set) add(v uint32) { s[v] = struct{}{} }
func (s set) has(v uint32) bool { _, ok := s[v]; return ok }
// bextr performs a bitfield extract on v. start should be the LSB of the field
// you wish to extract, and length the number of bits to extract.
//
// For example: start=0 and length=4 for the following 64-bit word would result
// in 1111 being returned.
//
// <snip 56 bits>00011110
// returns 1110
func bextr(v uint64, start, length uint8) uint64 {
return (v >> start) & ((1 << length) - 1)
}
func bextr32(v uint32, start, length uint8) uint32 {
return (v >> start) & ((1 << length) - 1)
}

683
pkg/estimator/hll/hll_test.go Normal file
View File

@ -0,0 +1,683 @@
package hll
import (
crand "crypto/rand"
"encoding/binary"
"fmt"
"math"
"math/rand"
"reflect"
"testing"
"unsafe"
"github.com/davecgh/go-spew/spew"
)
func nopHash(buf []byte) uint64 {
if len(buf) != 8 {
panic(fmt.Sprintf("unexpected size buffer: %d", len(buf)))
}
return binary.BigEndian.Uint64(buf)
}
func toByte(v uint64) []byte {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], v)
return buf[:]
}
func TestPlus_Bytes(t *testing.T) {
testCases := []struct {
p uint8
normal bool
}{
{4, false},
{5, false},
{4, true},
{5, true},
}
for i, testCase := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
h := NewTestPlus(testCase.p)
plusStructOverhead := int(unsafe.Sizeof(*h))
compressedListOverhead := int(unsafe.Sizeof(*h.sparseList))
var expectedDenseListCapacity, expectedSparseListCapacity int
if testCase.normal {
h.toNormal()
// denseList has capacity for 2^p elements, one byte each
expectedDenseListCapacity = int(math.Pow(2, float64(testCase.p)))
if expectedDenseListCapacity != cap(h.denseList) {
t.Errorf("denseList capacity: want %d got %d", expectedDenseListCapacity, cap(h.denseList))
}
} else {
// sparseList has capacity for 2^p elements, one byte each
expectedSparseListCapacity = int(math.Pow(2, float64(testCase.p)))
if expectedSparseListCapacity != cap(h.sparseList.b) {
t.Errorf("sparseList capacity: want %d got %d", expectedSparseListCapacity, cap(h.sparseList.b))
}
expectedSparseListCapacity += compressedListOverhead
}
expectedSize := plusStructOverhead + expectedDenseListCapacity + expectedSparseListCapacity
if expectedSize != h.Bytes() {
t.Errorf("Bytes(): want %d got %d", expectedSize, h.Bytes())
}
})
}
}
func TestPlus_Add_NoSparse(t *testing.T) {
h := NewTestPlus(16)
h.toNormal()
h.Add(toByte(0x00010fffffffffff))
n := h.denseList[1]
if n != 5 {
t.Error(n)
}
h.Add(toByte(0x0002ffffffffffff))
n = h.denseList[2]
if n != 1 {
t.Error(n)
}
h.Add(toByte(0x0003000000000000))
n = h.denseList[3]
if n != 49 {
t.Error(n)
}
h.Add(toByte(0x0003000000000001))
n = h.denseList[3]
if n != 49 {
t.Error(n)
}
h.Add(toByte(0xff03700000000000))
n = h.denseList[0xff03]
if n != 2 {
t.Error(n)
}
h.Add(toByte(0xff03080000000000))
n = h.denseList[0xff03]
if n != 5 {
t.Error(n)
}
}
func TestPlusPrecision_NoSparse(t *testing.T) {
h := NewTestPlus(4)
h.toNormal()
h.Add(toByte(0x1fffffffffffffff))
n := h.denseList[1]
if n != 1 {
t.Error(n)
}
h.Add(toByte(0xffffffffffffffff))
n = h.denseList[0xf]
if n != 1 {
t.Error(n)
}
h.Add(toByte(0x00ffffffffffffff))
n = h.denseList[0]
if n != 5 {
t.Error(n)
}
}
func TestPlus_toNormal(t *testing.T) {
h := NewTestPlus(16)
h.Add(toByte(0x00010fffffffffff))
h.toNormal()
c := h.Count()
if c != 1 {
t.Error(c)
}
if h.sparse {
t.Error("toNormal should convert to normal")
}
h = NewTestPlus(16)
h.hash = nopHash
h.Add(toByte(0x00010fffffffffff))
h.Add(toByte(0x0002ffffffffffff))
h.Add(toByte(0x0003000000000000))
h.Add(toByte(0x0003000000000001))
h.Add(toByte(0xff03700000000000))
h.Add(toByte(0xff03080000000000))
h.mergeSparse()
h.toNormal()
n := h.denseList[1]
if n != 5 {
t.Error(n)
}
n = h.denseList[2]
if n != 1 {
t.Error(n)
}
n = h.denseList[3]
if n != 49 {
t.Error(n)
}
n = h.denseList[0xff03]
if n != 5 {
t.Error(n)
}
}
func TestPlusCount(t *testing.T) {
h := NewTestPlus(16)
n := h.Count()
if n != 0 {
t.Error(n)
}
h.Add(toByte(0x00010fffffffffff))
h.Add(toByte(0x00020fffffffffff))
h.Add(toByte(0x00030fffffffffff))
h.Add(toByte(0x00040fffffffffff))
h.Add(toByte(0x00050fffffffffff))
h.Add(toByte(0x00050fffffffffff))
n = h.Count()
if n != 5 {
t.Error(n)
}
// not mutated, still returns correct count
n = h.Count()
if n != 5 {
t.Error(n)
}
h.Add(toByte(0x00060fffffffffff))
// mutated
n = h.Count()
if n != 6 {
t.Error(n)
}
}
func TestPlus_Merge_Error(t *testing.T) {
h := NewTestPlus(16)
h2 := NewTestPlus(10)
err := h.Merge(h2)
if err == nil {
t.Error("different precision should return error")
}
}
func TestHLL_Merge_Sparse(t *testing.T) {
h := NewTestPlus(16)
h.Add(toByte(0x00010fffffffffff))
h.Add(toByte(0x00020fffffffffff))
h.Add(toByte(0x00030fffffffffff))
h.Add(toByte(0x00040fffffffffff))
h.Add(toByte(0x00050fffffffffff))
h.Add(toByte(0x00050fffffffffff))
h2 := NewTestPlus(16)
h2.Merge(h)
n := h2.Count()
if n != 5 {
t.Error(n)
}
if h2.sparse {
t.Error("Merge should convert to normal")
}
if !h.sparse {
t.Error("Merge should not modify argument")
}
h2.Merge(h)
n = h2.Count()
if n != 5 {
t.Error(n)
}
h.Add(toByte(0x00060fffffffffff))
h.Add(toByte(0x00070fffffffffff))
h.Add(toByte(0x00080fffffffffff))
h.Add(toByte(0x00090fffffffffff))
h.Add(toByte(0x000a0fffffffffff))
h.Add(toByte(0x000a0fffffffffff))
n = h.Count()
if n != 10 {
t.Error(n)
}
h2.Merge(h)
n = h2.Count()
if n != 10 {
t.Error(n)
}
}
func TestHLL_Merge_Normal(t *testing.T) {
h := NewTestPlus(16)
h.toNormal()
h.Add(toByte(0x00010fffffffffff))
h.Add(toByte(0x00020fffffffffff))
h.Add(toByte(0x00030fffffffffff))
h.Add(toByte(0x00040fffffffffff))
h.Add(toByte(0x00050fffffffffff))
h.Add(toByte(0x00050fffffffffff))
h2 := NewTestPlus(16)
h2.toNormal()
h2.Merge(h)
n := h2.Count()
if n != 5 {
t.Error(n)
}
h2.Merge(h)
n = h2.Count()
if n != 5 {
t.Error(n)
}
h.Add(toByte(0x00060fffffffffff))
h.Add(toByte(0x00070fffffffffff))
h.Add(toByte(0x00080fffffffffff))
h.Add(toByte(0x00090fffffffffff))
h.Add(toByte(0x000a0fffffffffff))
h.Add(toByte(0x000a0fffffffffff))
n = h.Count()
if n != 10 {
t.Error(n)
}
h2.Merge(h)
n = h2.Count()
if n != 10 {
t.Error(n)
}
}
func TestPlus_Merge(t *testing.T) {
h := NewTestPlus(16)
k1 := uint64(0xf000017000000000)
h.Add(toByte(k1))
if !h.tmpSet.has(h.encodeHash(k1)) {
t.Error("key not in hash")
}
k2 := uint64(0x000fff8f00000000)
h.Add(toByte(k2))
if !h.tmpSet.has(h.encodeHash(k2)) {
t.Error("key not in hash")
}
if len(h.tmpSet) != 2 {
t.Error(h.tmpSet)
}
h.mergeSparse()
if len(h.tmpSet) != 0 {
t.Error(h.tmpSet)
}
if h.sparseList.count != 2 {
t.Error(h.sparseList)
}
iter := h.sparseList.Iter()
n := iter.Next()
if n != h.encodeHash(k2) {
t.Error(n)
}
n = iter.Next()
if n != h.encodeHash(k1) {
t.Error(n)
}
k3 := uint64(0x0f00017000000000)
h.Add(toByte(k3))
if !h.tmpSet.has(h.encodeHash(k3)) {
t.Error("key not in hash")
}
h.mergeSparse()
if len(h.tmpSet) != 0 {
t.Error(h.tmpSet)
}
if h.sparseList.count != 3 {
t.Error(h.sparseList)
}
iter = h.sparseList.Iter()
n = iter.Next()
if n != h.encodeHash(k2) {
t.Error(n)
}
n = iter.Next()
if n != h.encodeHash(k3) {
t.Error(n)
}
n = iter.Next()
if n != h.encodeHash(k1) {
t.Error(n)
}
h.Add(toByte(k1))
if !h.tmpSet.has(h.encodeHash(k1)) {
t.Error("key not in hash")
}
h.mergeSparse()
if len(h.tmpSet) != 0 {
t.Error(h.tmpSet)
}
if h.sparseList.count != 3 {
t.Error(h.sparseList)
}
iter = h.sparseList.Iter()
n = iter.Next()
if n != h.encodeHash(k2) {
t.Error(n)
}
n = iter.Next()
if n != h.encodeHash(k3) {
t.Error(n)
}
n = iter.Next()
if n != h.encodeHash(k1) {
t.Error(n)
}
}
func TestPlus_EncodeDecode(t *testing.T) {
h := NewTestPlus(8)
i, r := h.decodeHash(h.encodeHash(0xffffff8000000000))
if i != 0xff {
t.Error(i)
}
if r != 1 {
t.Error(r)
}
i, r = h.decodeHash(h.encodeHash(0xff00000000000000))
if i != 0xff {
t.Error(i)
}
if r != 57 {
t.Error(r)
}
i, r = h.decodeHash(h.encodeHash(0xff30000000000000))
if i != 0xff {
t.Error(i)
}
if r != 3 {
t.Error(r)
}
i, r = h.decodeHash(h.encodeHash(0xaa10000000000000))
if i != 0xaa {
t.Error(i)
}
if r != 4 {
t.Error(r)
}
i, r = h.decodeHash(h.encodeHash(0xaa0f000000000000))
if i != 0xaa {
t.Error(i)
}
if r != 5 {
t.Error(r)
}
}
func TestPlus_Error(t *testing.T) {
_, err := NewPlus(3)
if err == nil {
t.Error("precision 3 should return error")
}
_, err = NewPlus(18)
if err != nil {
t.Error(err)
}
_, err = NewPlus(19)
if err == nil {
t.Error("precision 17 should return error")
}
}
func TestPlus_Marshal_Unmarshal_Sparse(t *testing.T) {
h, _ := NewPlus(4)
h.sparse = true
h.tmpSet = map[uint32]struct{}{26: struct{}{}, 40: struct{}{}}
// Add a bunch of values to the sparse representation.
for i := 0; i < 10; i++ {
h.sparseList.Append(uint32(rand.Int()))
}
data, err := h.MarshalBinary()
if err != nil {
t.Fatal(err)
}
// Peeking at the first byte should reveal the version.
if got, exp := data[0], byte(2); got != exp {
t.Fatalf("got byte %v, expected %v", got, exp)
}
var res Plus
if err := res.UnmarshalBinary(data); err != nil {
t.Fatal(err)
}
// reflect.DeepEqual will always return false when comparing non-nil
// functions, so we'll set them to nil.
h.hash, res.hash = nil, nil
if got, exp := &res, h; !reflect.DeepEqual(got, exp) {
t.Fatalf("got %v, wanted %v", spew.Sdump(got), spew.Sdump(exp))
}
}
func TestPlus_Marshal_Unmarshal_Dense(t *testing.T) {
h, _ := NewPlus(4)
h.sparse = false
// Add a bunch of values to the dense representation.
for i := 0; i < 10; i++ {
h.denseList = append(h.denseList, uint8(rand.Int()))
}
data, err := h.MarshalBinary()
if err != nil {
t.Fatal(err)
}
// Peeking at the first byte should reveal the version.
if got, exp := data[0], byte(2); got != exp {
t.Fatalf("got byte %v, expected %v", got, exp)
}
var res Plus
if err := res.UnmarshalBinary(data); err != nil {
t.Fatal(err)
}
// reflect.DeepEqual will always return false when comparing non-nil
// functions, so we'll set them to nil.
h.hash, res.hash = nil, nil
if got, exp := &res, h; !reflect.DeepEqual(got, exp) {
t.Fatalf("got %v, wanted %v", spew.Sdump(got), spew.Sdump(exp))
}
}
// Tests that a sketch can be serialised / unserialised and keep an accurate
// cardinality estimate.
func TestPlus_Marshal_Unmarshal_Count(t *testing.T) {
if testing.Short() {
t.Skip("Skipping test in short mode")
}
count := make(map[string]struct{}, 1000000)
h, _ := NewPlus(16)
buf := make([]byte, 8)
for i := 0; i < 1000000; i++ {
if _, err := crand.Read(buf); err != nil {
panic(err)
}
count[string(buf)] = struct{}{}
// Add to the sketch.
h.Add(buf)
}
gotC := h.Count()
epsilon := 15000 // 1.5%
if got, exp := math.Abs(float64(int(gotC)-len(count))), epsilon; int(got) > exp {
t.Fatalf("error was %v for estimation %d and true cardinality %d", got, gotC, len(count))
}
// Serialise the sketch.
sketch, err := h.MarshalBinary()
if err != nil {
t.Fatal(err)
}
// Deserialise.
h = &Plus{}
if err := h.UnmarshalBinary(sketch); err != nil {
t.Fatal(err)
}
// The count should be the same
oldC := gotC
if got, exp := h.Count(), oldC; got != exp {
t.Fatalf("got %d, expected %d", got, exp)
}
// Add some more values.
for i := 0; i < 1000000; i++ {
if _, err := crand.Read(buf); err != nil {
panic(err)
}
count[string(buf)] = struct{}{}
// Add to the sketch.
h.Add(buf)
}
// The sketch should still be working correctly.
gotC = h.Count()
epsilon = 30000 // 1.5%
if got, exp := math.Abs(float64(int(gotC)-len(count))), epsilon; int(got) > exp {
t.Fatalf("error was %v for estimation %d and true cardinality %d", got, gotC, len(count))
}
}
func NewTestPlus(p uint8) *Plus {
h, err := NewPlus(p)
if err != nil {
panic(err)
}
h.hash = nopHash
return h
}
// Generate random data to add to the sketch.
func genData(n int) [][]byte {
out := make([][]byte, 0, n)
buf := make([]byte, 8)
for i := 0; i < n; i++ {
// generate 8 random bytes
n, err := rand.Read(buf)
if err != nil {
panic(err)
} else if n != 8 {
panic(fmt.Errorf("only %d bytes generated", n))
}
out = append(out, buf)
}
if len(out) != n {
panic(fmt.Sprintf("wrong size slice: %d", n))
}
return out
}
// Memoises values to be added to a sketch during a benchmark.
var benchdata = map[int][][]byte{}
func benchmarkPlusAdd(b *testing.B, h *Plus, n int) {
blobs, ok := benchdata[n]
if !ok {
// Generate it.
benchdata[n] = genData(n)
blobs = benchdata[n]
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
for j := 0; j < len(blobs); j++ {
h.Add(blobs[j])
}
}
b.StopTimer()
}
func BenchmarkPlus_Add_100(b *testing.B) {
h, _ := NewPlus(16)
benchmarkPlusAdd(b, h, 100)
}
func BenchmarkPlus_Add_1000(b *testing.B) {
h, _ := NewPlus(16)
benchmarkPlusAdd(b, h, 1000)
}
func BenchmarkPlus_Add_10000(b *testing.B) {
h, _ := NewPlus(16)
benchmarkPlusAdd(b, h, 10000)
}
func BenchmarkPlus_Add_100000(b *testing.B) {
h, _ := NewPlus(16)
benchmarkPlusAdd(b, h, 100000)
}
func BenchmarkPlus_Add_1000000(b *testing.B) {
h, _ := NewPlus(16)
benchmarkPlusAdd(b, h, 1000000)
}
func BenchmarkPlus_Add_10000000(b *testing.B) {
h, _ := NewPlus(16)
benchmarkPlusAdd(b, h, 10000000)
}
func BenchmarkPlus_Add_100000000(b *testing.B) {
h, _ := NewPlus(16)
benchmarkPlusAdd(b, h, 100000000)
}

24
pkg/estimator/sketch.go Normal file
View File

@ -0,0 +1,24 @@
package estimator
import "encoding"
// Sketch is the interface representing a sketch for estimating cardinality.
type Sketch interface {
// Add adds a single value to the sketch.
Add(v []byte)
// Count returns a cardinality estimate for the sketch.
Count() uint64
// Merge merges another sketch into this one.
Merge(s Sketch) error
// Bytes estimates the memory footprint of the sketch, in bytes.
Bytes() int
// Clone returns a deep copy of the sketch.
Clone() Sketch
encoding.BinaryMarshaler
encoding.BinaryUnmarshaler
}

35
pkg/file/file_unix.go Normal file
View File

@ -0,0 +1,35 @@
// +build !windows
package file
import (
"os"
"syscall"
)
func SyncDir(dirName string) error {
// fsync the dir to flush the rename
dir, err := os.OpenFile(dirName, os.O_RDONLY, os.ModeDir)
if err != nil {
return err
}
defer dir.Close()
// While we're on unix, we may be running in a Docker container that is
// pointed at a Windows volume over samba. That doesn't support fsyncs
// on directories. This shows itself as an EINVAL, so we ignore that
// error.
err = dir.Sync()
if pe, ok := err.(*os.PathError); ok && pe.Err == syscall.EINVAL {
err = nil
} else if err != nil {
return err
}
return dir.Close()
}
// RenameFile will rename the source to target using os function.
func RenameFile(oldpath, newpath string) error {
return os.Rename(oldpath, newpath)
}

18
pkg/file/file_windows.go Normal file
View File

@ -0,0 +1,18 @@
package file
import "os"
func SyncDir(dirName string) error {
return nil
}
// RenameFile will rename the source to target using os function. If target exists it will be removed before renaming.
func RenameFile(oldpath, newpath string) error {
if _, err := os.Stat(newpath); err == nil {
if err = os.Remove(newpath); nil != err {
return err
}
}
return os.Rename(oldpath, newpath)
}

31
pkg/radix/buffer.go Normal file
View File

@ -0,0 +1,31 @@
package radix
// bufferSize is the size of the buffer and the largest slice that can be
// contained in it.
const bufferSize = 4096
// buffer is a type that amoritizes allocations into larger ones, handing out
// small subslices to make copies.
type buffer []byte
// Copy returns a copy of the passed in byte slice allocated using the byte
// slice in the buffer.
func (b *buffer) Copy(x []byte) []byte {
// if we can never have enough room, just return a copy
if len(x) > bufferSize {
out := make([]byte, len(x))
copy(out, x)
return out
}
// if we don't have enough room, reallocate the buf first
if len(x) > len(*b) {
*b = make([]byte, bufferSize)
}
// create a copy and hand out a slice
copy(*b, x)
out := (*b)[:len(x):len(x)]
*b = (*b)[len(x):]
return out
}

55
pkg/radix/buffer_test.go Normal file
View File

@ -0,0 +1,55 @@
package radix
import (
"bytes"
"math/rand"
"testing"
)
func TestBuffer(t *testing.T) {
var buf buffer
for i := 0; i < 1000; i++ {
x1 := make([]byte, rand.Intn(32)+1)
for j := range x1 {
x1[j] = byte(i + j)
}
x2 := buf.Copy(x1)
if !bytes.Equal(x2, x1) {
t.Fatal("bad copy")
}
x1[0] += 1
if bytes.Equal(x2, x1) {
t.Fatal("bad copy")
}
}
}
func TestBufferAppend(t *testing.T) {
var buf buffer
x1 := buf.Copy(make([]byte, 1))
x2 := buf.Copy(make([]byte, 1))
_ = append(x1, 1)
if x2[0] != 0 {
t.Fatal("append wrote past")
}
}
func TestBufferLarge(t *testing.T) {
var buf buffer
x1 := make([]byte, bufferSize+1)
x2 := buf.Copy(x1)
if !bytes.Equal(x1, x2) {
t.Fatal("bad copy")
}
x1[0] += 1
if bytes.Equal(x1, x2) {
t.Fatal("bad copy")
}
}

92
pkg/radix/sort.go Normal file
View File

@ -0,0 +1,92 @@
// Portions of this file from github.com/shawnsmithdev/zermelo under the MIT license.
//
// The MIT License (MIT)
//
// Copyright (c) 2014 Shawn Smith
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package radix
import (
"sort"
)
const (
minSize = 256
radix uint = 8
bitSize uint = 64
)
// SortUint64s sorts a slice of uint64s.
func SortUint64s(x []uint64) {
if len(x) < 2 {
return
} else if len(x) < minSize {
sort.Slice(x, func(i, j int) bool { return x[i] < x[j] })
} else {
doSort(x)
}
}
func doSort(x []uint64) {
// Each pass processes a byte offset, copying back and forth between slices
from := x
to := make([]uint64, len(x))
var key uint8
var offset [256]int // Keep track of where groups start
for keyOffset := uint(0); keyOffset < bitSize; keyOffset += radix {
keyMask := uint64(0xFF << keyOffset) // Current 'digit' to look at
var counts [256]int // Keep track of the number of elements for each kind of byte
sorted := true // Check for already sorted
prev := uint64(0) // if elem is always >= prev it is already sorted
for _, elem := range from {
key = uint8((elem & keyMask) >> keyOffset) // fetch the byte at current 'digit'
counts[key]++ // count of elems to put in this digit's bucket
if sorted { // Detect sorted
sorted = elem >= prev
prev = elem
}
}
if sorted { // Short-circuit sorted
if (keyOffset/radix)%2 == 1 {
copy(to, from)
}
return
}
// Find target bucket offsets
offset[0] = 0
for i := 1; i < len(offset); i++ {
offset[i] = offset[i-1] + counts[i-1]
}
// Rebucket while copying to other buffer
for _, elem := range from {
key = uint8((elem & keyMask) >> keyOffset) // Get the digit
to[offset[key]] = elem // Copy the element to the digit's bucket
offset[key]++ // One less space, move the offset
}
// On next pass copy data the other way
to, from = from, to
}
}

27
pkg/radix/sort_test.go Normal file
View File

@ -0,0 +1,27 @@
package radix
import (
"math/rand"
"testing"
)
func benchmarkSort(b *testing.B, size int) {
orig := make([]uint64, size)
for i := range orig {
orig[i] = uint64(rand.Int63())
}
data := make([]uint64, size)
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
copy(data, orig)
SortUint64s(data)
}
}
func BenchmarkSort_64(b *testing.B) { benchmarkSort(b, 64) }
func BenchmarkSort_128(b *testing.B) { benchmarkSort(b, 128) }
func BenchmarkSort_256(b *testing.B) { benchmarkSort(b, 256) }
func BenchmarkSort_12K(b *testing.B) { benchmarkSort(b, 12*1024) }

428
pkg/radix/tree.go Normal file
View File

@ -0,0 +1,428 @@
package radix
// This is a fork of https://github.com/armon/go-radix that removes the
// ability to update nodes as well as uses fixed int value type.
import (
"bytes"
"sort"
"sync"
)
// leafNode is used to represent a value
type leafNode struct {
valid bool // true if key/val are valid
key []byte
val int
}
// edge is used to represent an edge node
type edge struct {
label byte
node *node
}
type node struct {
// leaf is used to store possible leaf
leaf leafNode
// prefix is the common prefix we ignore
prefix []byte
// Edges should be stored in-order for iteration.
// We avoid a fully materialized slice to save memory,
// since in most cases we expect to be sparse
edges edges
}
func (n *node) isLeaf() bool {
return n.leaf.valid
}
func (n *node) addEdge(e edge) {
// find the insertion point with bisection
num := len(n.edges)
i, j := 0, num
for i < j {
h := int(uint(i+j) >> 1)
if n.edges[h].label < e.label {
i = h + 1
} else {
j = h
}
}
// make room, copy the suffix, and insert.
n.edges = append(n.edges, edge{})
copy(n.edges[i+1:], n.edges[i:])
n.edges[i] = e
}
func (n *node) replaceEdge(e edge) {
num := len(n.edges)
idx := sort.Search(num, func(i int) bool {
return n.edges[i].label >= e.label
})
if idx < num && n.edges[idx].label == e.label {
n.edges[idx].node = e.node
return
}
panic("replacing missing edge")
}
func (n *node) getEdge(label byte) *node {
// linear search for small slices
if len(n.edges) < 16 {
for _, e := range n.edges {
if e.label == label {
return e.node
}
}
return nil
}
// binary search for larger
num := len(n.edges)
i, j := 0, num
for i < j {
h := int(uint(i+j) >> 1)
if n.edges[h].label < label {
i = h + 1
} else {
j = h
}
}
if i < num && n.edges[i].label == label {
return n.edges[i].node
}
return nil
}
type edges []edge
// Tree implements a radix tree. This can be treated as a
// Dictionary abstract data type. The main advantage over
// a standard hash map is prefix-based lookups and
// ordered iteration. The tree is safe for concurrent access.
type Tree struct {
mu sync.RWMutex
root *node
size int
buf buffer
}
// New returns an empty Tree
func New() *Tree {
return &Tree{root: &node{}}
}
// NewFromMap returns a new tree containing the keys
// from an existing map
func NewFromMap(m map[string]int) *Tree {
t := &Tree{root: &node{}}
for k, v := range m {
t.Insert([]byte(k), v)
}
return t
}
// Len is used to return the number of elements in the tree
func (t *Tree) Len() int {
t.mu.RLock()
size := t.size
t.mu.RUnlock()
return size
}
// longestPrefix finds the length of the shared prefix
// of two strings
func longestPrefix(k1, k2 []byte) int {
// for loops can't be inlined, but goto's can. we also use uint to help
// out the compiler to prove bounds checks aren't necessary on the index
// operations.
lk1, lk2 := uint(len(k1)), uint(len(k2))
i := uint(0)
loop:
if lk1 <= i || lk2 <= i {
return int(i)
}
if k1[i] != k2[i] {
return int(i)
}
i++
goto loop
}
// Insert is used to add a newentry or update
// an existing entry. Returns if inserted.
func (t *Tree) Insert(s []byte, v int) (int, bool) {
t.mu.RLock()
var parent *node
n := t.root
search := s
for {
// Handle key exhaution
if len(search) == 0 {
if n.isLeaf() {
old := n.leaf.val
t.mu.RUnlock()
return old, false
}
n.leaf = leafNode{
key: t.buf.Copy(s),
val: v,
valid: true,
}
t.size++
t.mu.RUnlock()
return v, true
}
// Look for the edge
parent = n
n = n.getEdge(search[0])
// No edge, create one
if n == nil {
newNode := &node{
leaf: leafNode{
key: t.buf.Copy(s),
val: v,
valid: true,
},
prefix: t.buf.Copy(search),
}
e := edge{
label: search[0],
node: newNode,
}
parent.addEdge(e)
t.size++
t.mu.RUnlock()
return v, true
}
// Determine longest prefix of the search key on match
commonPrefix := longestPrefix(search, n.prefix)
if commonPrefix == len(n.prefix) {
search = search[commonPrefix:]
continue
}
// Split the node
t.size++
child := &node{
prefix: t.buf.Copy(search[:commonPrefix]),
}
parent.replaceEdge(edge{
label: search[0],
node: child,
})
// Restore the existing node
child.addEdge(edge{
label: n.prefix[commonPrefix],
node: n,
})
n.prefix = n.prefix[commonPrefix:]
// Create a new leaf node
leaf := leafNode{
key: t.buf.Copy(s),
val: v,
valid: true,
}
// If the new key is a subset, add to to this node
search = search[commonPrefix:]
if len(search) == 0 {
child.leaf = leaf
t.mu.RUnlock()
return v, true
}
// Create a new edge for the node
child.addEdge(edge{
label: search[0],
node: &node{
leaf: leaf,
prefix: t.buf.Copy(search),
},
})
t.mu.RUnlock()
return v, true
}
}
// DeletePrefix is used to delete the subtree under a prefix
// Returns how many nodes were deleted
// Use this to delete large subtrees efficiently
func (t *Tree) DeletePrefix(s []byte) int {
t.mu.Lock()
defer t.mu.Unlock()
return t.deletePrefix(nil, t.root, s)
}
// delete does a recursive deletion
func (t *Tree) deletePrefix(parent, n *node, prefix []byte) int {
// Check for key exhaustion
if len(prefix) == 0 {
// Remove the leaf node
subTreeSize := 0
//recursively walk from all edges of the node to be deleted
recursiveWalk(n, func(s []byte, v int) bool {
subTreeSize++
return false
})
if n.isLeaf() {
n.leaf = leafNode{}
}
n.edges = nil // deletes the entire subtree
// Check if we should merge the parent's other child
if parent != nil && parent != t.root && len(parent.edges) == 1 && !parent.isLeaf() {
parent.mergeChild()
}
t.size -= subTreeSize
return subTreeSize
}
// Look for an edge
label := prefix[0]
child := n.getEdge(label)
if child == nil || (!bytes.HasPrefix(child.prefix, prefix) && !bytes.HasPrefix(prefix, child.prefix)) {
return 0
}
// Consume the search prefix
if len(child.prefix) > len(prefix) {
prefix = prefix[len(prefix):]
} else {
prefix = prefix[len(child.prefix):]
}
return t.deletePrefix(n, child, prefix)
}
func (n *node) mergeChild() {
e := n.edges[0]
child := e.node
prefix := make([]byte, 0, len(n.prefix)+len(child.prefix))
prefix = append(prefix, n.prefix...)
prefix = append(prefix, child.prefix...)
n.prefix = prefix
n.leaf = child.leaf
n.edges = child.edges
}
// Get is used to lookup a specific key, returning
// the value and if it was found
func (t *Tree) Get(s []byte) (int, bool) {
t.mu.RLock()
n := t.root
search := s
for {
// Check for key exhaution
if len(search) == 0 {
if n.isLeaf() {
t.mu.RUnlock()
return n.leaf.val, true
}
break
}
// Look for an edge
n = n.getEdge(search[0])
if n == nil {
break
}
// Consume the search prefix
if bytes.HasPrefix(search, n.prefix) {
search = search[len(n.prefix):]
} else {
break
}
}
t.mu.RUnlock()
return 0, false
}
// walkFn is used when walking the tree. Takes a
// key and value, returning if iteration should
// be terminated.
type walkFn func(s []byte, v int) bool
// recursiveWalk is used to do a pre-order walk of a node
// recursively. Returns true if the walk should be aborted
func recursiveWalk(n *node, fn walkFn) bool {
// Visit the leaf values if any
if n.leaf.valid && fn(n.leaf.key, n.leaf.val) {
return true
}
// Recurse on the children
for _, e := range n.edges {
if recursiveWalk(e.node, fn) {
return true
}
}
return false
}
// Minimum is used to return the minimum value in the tree
func (t *Tree) Minimum() ([]byte, int, bool) {
t.mu.RLock()
n := t.root
for {
if n.isLeaf() {
t.mu.RUnlock()
return n.leaf.key, n.leaf.val, true
}
if len(n.edges) > 0 {
n = n.edges[0].node
} else {
break
}
}
t.mu.RUnlock()
return nil, 0, false
}
// Maximum is used to return the maximum value in the tree
func (t *Tree) Maximum() ([]byte, int, bool) {
t.mu.RLock()
n := t.root
for {
if num := len(n.edges); num > 0 {
n = n.edges[num-1].node
continue
}
if n.isLeaf() {
t.mu.RUnlock()
return n.leaf.key, n.leaf.val, true
}
break
}
t.mu.RUnlock()
return nil, 0, false
}

174
pkg/radix/tree_test.go Normal file
View File

@ -0,0 +1,174 @@
package radix
import (
"crypto/rand"
"fmt"
"reflect"
"testing"
)
// generateUUID is used to generate a random UUID
func generateUUID() string {
buf := make([]byte, 16)
if _, err := rand.Read(buf); err != nil {
panic(fmt.Errorf("failed to read random bytes: %v", err))
}
return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x",
buf[0:4],
buf[4:6],
buf[6:8],
buf[8:10],
buf[10:16])
}
func TestRadix(t *testing.T) {
var min, max string
inp := make(map[string]int)
for i := 0; i < 1000; i++ {
gen := generateUUID()
inp[gen] = i
if gen < min || i == 0 {
min = gen
}
if gen > max || i == 0 {
max = gen
}
}
r := NewFromMap(inp)
if r.Len() != len(inp) {
t.Fatalf("bad length: %v %v", r.Len(), len(inp))
}
// Check min and max
outMin, _, _ := r.Minimum()
if string(outMin) != min {
t.Fatalf("bad minimum: %s %v", outMin, min)
}
outMax, _, _ := r.Maximum()
if string(outMax) != max {
t.Fatalf("bad maximum: %s %v", outMax, max)
}
for k, v := range inp {
out, ok := r.Get([]byte(k))
if !ok {
t.Fatalf("missing key: %v", k)
}
if out != v {
t.Fatalf("value mis-match: %v %v", out, v)
}
}
}
func TestDeletePrefix(t *testing.T) {
type exp struct {
inp []string
prefix string
out []string
numDeleted int
}
cases := []exp{
{[]string{"", "A", "AB", "ABC", "R", "S"}, "A", []string{"", "R", "S"}, 3},
{[]string{"", "A", "AB", "ABC", "R", "S"}, "ABC", []string{"", "A", "AB", "R", "S"}, 1},
{[]string{"", "A", "AB", "ABC", "R", "S"}, "", []string{}, 6},
{[]string{"", "A", "AB", "ABC", "R", "S"}, "S", []string{"", "A", "AB", "ABC", "R"}, 1},
{[]string{"", "A", "AB", "ABC", "R", "S"}, "SS", []string{"", "A", "AB", "ABC", "R", "S"}, 0},
}
for _, test := range cases {
r := New()
for _, ss := range test.inp {
r.Insert([]byte(ss), 1)
}
deleted := r.DeletePrefix([]byte(test.prefix))
if deleted != test.numDeleted {
t.Fatalf("Bad delete, expected %v to be deleted but got %v", test.numDeleted, deleted)
}
out := []string{}
fn := func(s []byte, v int) bool {
out = append(out, string(s))
return false
}
recursiveWalk(r.root, fn)
if !reflect.DeepEqual(out, test.out) {
t.Fatalf("mis-match: %v %v", out, test.out)
}
}
}
func TestInsert_Duplicate(t *testing.T) {
r := New()
vv, ok := r.Insert([]byte("cpu"), 1)
if vv != 1 {
t.Fatalf("value mismatch: got %v, exp %v", vv, 1)
}
if !ok {
t.Fatalf("value mismatch: got %v, exp %v", ok, true)
}
// Insert a dup with a different type should fail
vv, ok = r.Insert([]byte("cpu"), 2)
if vv != 1 {
t.Fatalf("value mismatch: got %v, exp %v", vv, 1)
}
if ok {
t.Fatalf("value mismatch: got %v, exp %v", ok, false)
}
}
//
// benchmarks
//
func BenchmarkTree_Insert(b *testing.B) {
t := New()
keys := make([][]byte, 0, 10000)
for i := 0; i < cap(keys); i++ {
k := []byte(fmt.Sprintf("cpu,host=%d", i))
if v, ok := t.Insert(k, 1); v != 1 || !ok {
b.Fatalf("insert failed: %v != 1 || !%v", v, ok)
}
keys = append(keys, k)
}
b.SetBytes(int64(len(keys)))
b.ReportAllocs()
b.ResetTimer()
for j := 0; j < b.N; j++ {
for _, key := range keys {
if v, ok := t.Insert(key, 1); v != 1 || ok {
b.Fatalf("insert failed: %v != 1 || !%v", v, ok)
}
}
}
}
func BenchmarkTree_InsertNew(b *testing.B) {
keys := make([][]byte, 0, 10000)
for i := 0; i < cap(keys); i++ {
k := []byte(fmt.Sprintf("cpu,host=%d", i))
keys = append(keys, k)
}
b.SetBytes(int64(len(keys)))
b.ReportAllocs()
b.ResetTimer()
for j := 0; j < b.N; j++ {
t := New()
for _, key := range keys {
t.Insert(key, 1)
}
}
}

2
pkg/slices/strings.go
View File

@ -1,5 +1,5 @@
// Package slices contains functions to operate on slices treated as sets.
package slices // import "github.com/influxdata/influxdb/pkg/slices"
package slices // import "github.com/influxdata/influxdb/v2/pkg/slices"
import "strings"

165
pkg/tar/stream.go Normal file
View File

@ -0,0 +1,165 @@
package tar
import (
"archive/tar"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/influxdata/influxdb/v2/pkg/file"
)
// Stream is a convenience function for creating a tar of a shard dir. It walks over the directory and subdirs,
// possibly writing each file to a tar writer stream. By default StreamFile is used, which will result in all files
// being written. A custom writeFunc can be passed so that each file may be written, modified+written, or skipped
// depending on the custom logic.
func Stream(w io.Writer, dir, relativePath string, writeFunc func(f os.FileInfo, shardRelativePath, fullPath string, tw *tar.Writer) error) error {
tw := tar.NewWriter(w)
defer tw.Close()
if writeFunc == nil {
writeFunc = StreamFile
}
return filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
// Skip adding an entry for the root dir
if dir == path && f.IsDir() {
return nil
}
// Figure out the the full relative path including any sub-dirs
subDir, _ := filepath.Split(path)
subDir, err = filepath.Rel(dir, subDir)
if err != nil {
return err
}
return writeFunc(f, filepath.Join(relativePath, subDir), path, tw)
})
}
// Generates a filtering function for Stream that checks an incoming file, and only writes the file to the stream if
// its mod time is later than since. Example: to tar only files newer than a certain datetime, use
// tar.Stream(w, dir, relativePath, SinceFilterTarFile(datetime))
func SinceFilterTarFile(since time.Time) func(f os.FileInfo, shardRelativePath, fullPath string, tw *tar.Writer) error {
return func(f os.FileInfo, shardRelativePath, fullPath string, tw *tar.Writer) error {
if f.ModTime().After(since) {
return StreamFile(f, shardRelativePath, fullPath, tw)
}
return nil
}
}
// stream a single file to tw, extending the header name using the shardRelativePath
func StreamFile(f os.FileInfo, shardRelativePath, fullPath string, tw *tar.Writer) error {
return StreamRenameFile(f, f.Name(), shardRelativePath, fullPath, tw)
}
/// Stream a single file to tw, using tarHeaderFileName instead of the actual filename
// e.g., when we want to write a *.tmp file using the original file's non-tmp name.
func StreamRenameFile(f os.FileInfo, tarHeaderFileName, relativePath, fullPath string, tw *tar.Writer) error {
h, err := tar.FileInfoHeader(f, f.Name())
if err != nil {
return err
}
h.Name = filepath.ToSlash(filepath.Join(relativePath, tarHeaderFileName))
if err := tw.WriteHeader(h); err != nil {
return err
}
if !f.Mode().IsRegular() {
return nil
}
fr, err := os.Open(fullPath)
if err != nil {
return err
}
defer fr.Close()
_, err = io.CopyN(tw, fr, h.Size)
return err
}
// Restore reads a tar archive from r and extracts all of its files into dir,
// using only the base name of each file.
func Restore(r io.Reader, dir string) error {
tr := tar.NewReader(r)
for {
if err := extractFile(tr, dir); err == io.EOF {
break
} else if err != nil {
return err
}
}
return file.SyncDir(dir)
}
// extractFile copies the next file from tr into dir, using the file's base name.
func extractFile(tr *tar.Reader, dir string) error {
// Read next archive file.
hdr, err := tr.Next()
if err != nil {
return err
}
// The hdr.Name is the relative path of the file from the root data dir.
// e.g (db/rp/1/xxxxx.tsm or db/rp/1/index/xxxxxx.tsi)
sections := strings.Split(filepath.FromSlash(hdr.Name), string(filepath.Separator))
if len(sections) < 3 {
return fmt.Errorf("invalid archive path: %s", hdr.Name)
}
relativePath := filepath.Join(sections[3:]...)
subDir, _ := filepath.Split(relativePath)
// If this is a directory entry (usually just `index` for tsi), create it an move on.
if hdr.Typeflag == tar.TypeDir {
return os.MkdirAll(filepath.Join(dir, subDir), os.FileMode(hdr.Mode).Perm())
}
// Make sure the dir we need to write into exists. It should, but just double check in
// case we get a slightly invalid tarball.
if subDir != "" {
if err := os.MkdirAll(filepath.Join(dir, subDir), 0755); err != nil {
return err
}
}
destPath := filepath.Join(dir, relativePath)
tmp := destPath + ".tmp"
// Create new file on disk.
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode).Perm())
if err != nil {
return err
}
defer f.Close()
// Copy from archive to the file.
if _, err := io.CopyN(f, tr, hdr.Size); err != nil {
return err
}
// Sync to disk & close.
if err := f.Sync(); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
return file.RenameFile(tmp, destPath)
}

30
pkg/tracing/context.go Normal file
View File

@ -0,0 +1,30 @@
package tracing
import "context"
type (
spanContextKey struct{}
traceContextKey struct{}
)
// NewContextWithSpan returns a new context with the given Span added.
func NewContextWithSpan(ctx context.Context, c *Span) context.Context {
return context.WithValue(ctx, spanContextKey{}, c)
}
// SpanFromContext returns the Span associated with ctx or nil if no Span has been assigned.
func SpanFromContext(ctx context.Context) *Span {
c, _ := ctx.Value(spanContextKey{}).(*Span)
return c
}
// NewContextWithTrace returns a new context with the given Trace added.
func NewContextWithTrace(ctx context.Context, t *Trace) context.Context {
return context.WithValue(ctx, traceContextKey{}, t)
}
// TraceFromContext returns the Trace associated with ctx or nil if no Trace has been assigned.
func TraceFromContext(ctx context.Context) *Trace {
c, _ := ctx.Value(traceContextKey{}).(*Trace)
return c
}

26
pkg/tracing/doc.go Normal file
View File

@ -0,0 +1,26 @@
/*
Package tracing provides a way for capturing hierarchical traces.
To start a new trace with a root span named select
trace, span := tracing.NewTrace("select")
It is recommended that a span be forwarded to callees using the
context package. Firstly, create a new context with the span associated
as follows
ctx = tracing.NewContextWithSpan(ctx, span)
followed by calling the API with the new context
SomeAPI(ctx, ...)
Once the trace is complete, it may be converted to a graph with the Tree method.
tree := t.Tree()
The tree is intended to be used with the Walk function in order to generate
different presentations. The default Tree#String method returns a tree.
*/
package tracing

117
pkg/tracing/fields/field.go Normal file
View File

@ -0,0 +1,117 @@
package fields
import (
"fmt"
"math"
"time"
)
type fieldType int
const (
stringType fieldType = iota
boolType
int64Type
uint64Type
durationType
float64Type
)
// Field instances are constructed via Bool, String, and so on.
//
// "heavily influenced by" (i.e., partially stolen from)
// https://github.com/opentracing/opentracing-go/log
type Field struct {
key string
fieldType fieldType
numericVal int64
stringVal string
}
// String adds a string-valued key:value pair to a Span.LogFields() record
func String(key, val string) Field {
return Field{
key: key,
fieldType: stringType,
stringVal: val,
}
}
// Bool adds a bool-valued key:value pair to a Span.LogFields() record
func Bool(key string, val bool) Field {
var numericVal int64
if val {
numericVal = 1
}
return Field{
key: key,
fieldType: boolType,
numericVal: numericVal,
}
}
/// Int64 adds an int64-valued key:value pair to a Span.LogFields() record
func Int64(key string, val int64) Field {
return Field{
key: key,
fieldType: int64Type,
numericVal: val,
}
}
// Uint64 adds a uint64-valued key:value pair to a Span.LogFields() record
func Uint64(key string, val uint64) Field {
return Field{
key: key,
fieldType: uint64Type,
numericVal: int64(val),
}
}
// Uint64 adds a uint64-valued key:value pair to a Span.LogFields() record
func Duration(key string, val time.Duration) Field {
return Field{
key: key,
fieldType: durationType,
numericVal: int64(val),
}
}
// Float64 adds a float64-valued key:value pair to a Span.LogFields() record
func Float64(key string, val float64) Field {
return Field{
key: key,
fieldType: float64Type,
numericVal: int64(math.Float64bits(val)),
}
}
// Key returns the field's key.
func (lf Field) Key() string {
return lf.key
}
// Value returns the field's value as interface{}.
func (lf Field) Value() interface{} {
switch lf.fieldType {
case stringType:
return lf.stringVal
case boolType:
return lf.numericVal != 0
case int64Type:
return int64(lf.numericVal)
case uint64Type:
return uint64(lf.numericVal)
case durationType:
return time.Duration(lf.numericVal)
case float64Type:
return math.Float64frombits(uint64(lf.numericVal))
default:
return nil
}
}
// String returns a string representation of the key and value.
func (lf Field) String() string {
return fmt.Sprint(lf.key, ": ", lf.Value())
}

61
pkg/tracing/fields/fields.go Normal file
View File

@ -0,0 +1,61 @@
package fields
import "sort"
type Fields []Field
// Merge merges other with the current set, replacing any matching keys from other.
func (fs *Fields) Merge(other Fields) {
var list []Field
i, j := 0, 0
for i < len(*fs) && j < len(other) {
if (*fs)[i].key < other[j].key {
list = append(list, (*fs)[i])
i++
} else if (*fs)[i].key > other[j].key {
list = append(list, other[j])
j++
} else {
// equal, then "other" replaces existing key
list = append(list, other[j])
i++
j++
}
}
if i < len(*fs) {
list = append(list, (*fs)[i:]...)
} else if j < len(other) {
list = append(list, other[j:]...)
}
*fs = list
}
// New creates a new set of fields, sorted by Key.
// Duplicate keys are removed.
func New(args ...Field) Fields {
fields := Fields(args)
sort.Slice(fields, func(i, j int) bool {
return fields[i].key < fields[j].key
})
// deduplicate
// loop invariant: fields[:i] has no duplicates
for i := 0; i < len(fields)-1; i++ {
j := i + 1
// find all duplicate keys
for j < len(fields) && fields[i].key == fields[j].key {
j++
}
d := (j - 1) - i // number of duplicate keys
if d > 0 {
// copy over duplicate keys in order to maintain loop invariant
copy(fields[i+1:], fields[j:])
fields = fields[:len(fields)-d]
}
}
return fields
}

101
pkg/tracing/fields/fields_test.go Normal file
View File

@ -0,0 +1,101 @@
package fields
import (
"testing"
"github.com/influxdata/influxdb/v2/pkg/testing/assert"
)
func makeFields(args ...string) Fields {
if len(args)%2 != 0 {
panic("uneven number of arguments")
}
var f Fields
for i := 0; i+1 < len(args); i += 2 {
f = append(f, String(args[i], args[i+1]))
}
return f
}
func TestNew(t *testing.T) {
cases := []struct {
n string
l []string
exp Fields
}{
{
n: "empty",
l: nil,
exp: makeFields(),
},
{
n: "not duplicates",
l: []string{"k01", "v01", "k03", "v03", "k02", "v02"},
exp: makeFields("k01", "v01", "k02", "v02", "k03", "v03"),
},
{
n: "duplicates at end",
l: []string{"k01", "v01", "k02", "v02", "k02", "v02"},
exp: makeFields("k01", "v01", "k02", "v02"),
},
{
n: "duplicates at start",
l: []string{"k01", "v01", "k02", "v02", "k01", "v01"},
exp: makeFields("k01", "v01", "k02", "v02"),
},
{
n: "duplicates in middle",
l: []string{"k01", "v01", "k02", "v02", "k03", "v03", "k02", "v02", "k02", "v02"},
exp: makeFields("k01", "v01", "k02", "v02", "k03", "v03"),
},
}
for _, tc := range cases {
t.Run(tc.n, func(t *testing.T) {
l := New(makeFields(tc.l...)...)
assert.Equal(t, tc.exp, l)
})
}
}
func TestFields_Merge(t *testing.T) {
cases := []struct {
n string
l, r Fields
exp Fields
}{
{
n: "no matching keys",
l: New(String("k05", "v05"), String("k03", "v03"), String("k01", "v01")),
r: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")),
exp: New(String("k05", "v05"), String("k03", "v03"), String("k01", "v01"), String("k02", "v02"), String("k04", "v04"), String("k00", "v00")),
},
{
n: "multiple matching keys",
l: New(String("k05", "v05"), String("k03", "v03"), String("k01", "v01")),
r: New(String("k02", "v02"), String("k03", "v03a"), String("k05", "v05a")),
exp: New(String("k05", "v05a"), String("k03", "v03a"), String("k01", "v01"), String("k02", "v02")),
},
{
n: "source empty",
l: New(),
r: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")),
exp: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")),
},
{
n: "other empty",
l: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")),
r: New(),
exp: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")),
},
}
for _, tc := range cases {
t.Run(tc.n, func(t *testing.T) {
l := tc.l
l.Merge(tc.r)
assert.Equal(t, tc.exp, l)
})
}
}

74
pkg/tracing/labels/labels.go Normal file
View File

@ -0,0 +1,74 @@
package labels
import "sort"
type Label struct {
Key, Value string
}
// The Labels type represents a set of labels, sorted by Key.
type Labels []Label
// Merge merges other with the current set, replacing any matching keys from other.
func (ls *Labels) Merge(other Labels) {
var list []Label
i, j := 0, 0
for i < len(*ls) && j < len(other) {
if (*ls)[i].Key < other[j].Key {
list = append(list, (*ls)[i])
i++
} else if (*ls)[i].Key > other[j].Key {
list = append(list, other[j])
j++
} else {
// equal, then "other" replaces existing key
list = append(list, other[j])
i++
j++
}
}
if i < len(*ls) {
list = append(list, (*ls)[i:]...)
} else if j < len(other) {
list = append(list, other[j:]...)
}
*ls = list
}
// New takes an even number of strings representing key-value pairs
// and creates a new slice of Labels. Duplicates are removed, however,
// there is no guarantee which will be removed
func New(args ...string) Labels {
if len(args)%2 != 0 {
panic("uneven number of arguments to label.Labels")
}
var labels Labels
for i := 0; i+1 < len(args); i += 2 {
labels = append(labels, Label{Key: args[i], Value: args[i+1]})
}
sort.Slice(labels, func(i, j int) bool {
return labels[i].Key < labels[j].Key
})
// deduplicate
// loop invariant: labels[:i] has no duplicates
for i := 0; i < len(labels)-1; i++ {
j := i + 1
// find all duplicate keys
for j < len(labels) && labels[i].Key == labels[j].Key {
j++
}
d := (j - 1) - i // number of duplicate keys
if d > 0 {
// copy over duplicate keys in order to maintain loop invariant
copy(labels[i+1:], labels[j:])
labels = labels[:len(labels)-d]
}
}
return labels
}

101
pkg/tracing/labels/labels_test.go Normal file
View File

@ -0,0 +1,101 @@
package labels
import (
"testing"
"github.com/influxdata/influxdb/v2/pkg/testing/assert"
)
func makeLabels(args ...string) Labels {
if len(args)%2 != 0 {
panic("uneven number of arguments")
}
var l Labels
for i := 0; i+1 < len(args); i += 2 {
l = append(l, Label{Key: args[i], Value: args[i+1]})
}
return l
}
func TestNew(t *testing.T) {
cases := []struct {
n string
l []string
exp Labels
}{
{
n: "empty",
l: nil,
exp: makeLabels(),
},
{
n: "not duplicates",
l: []string{"k01", "v01", "k03", "v03", "k02", "v02"},
exp: makeLabels("k01", "v01", "k02", "v02", "k03", "v03"),
},
{
n: "duplicates at end",
l: []string{"k01", "v01", "k02", "v02", "k02", "v02"},
exp: makeLabels("k01", "v01", "k02", "v02"),
},
{
n: "duplicates at start",
l: []string{"k01", "v01", "k02", "v02", "k01", "v01"},
exp: makeLabels("k01", "v01", "k02", "v02"),
},
{
n: "duplicates in middle",
l: []string{"k01", "v01", "k02", "v02", "k03", "v03", "k02", "v02", "k02", "v02"},
exp: makeLabels("k01", "v01", "k02", "v02", "k03", "v03"),
},
}
for _, tc := range cases {
t.Run(tc.n, func(t *testing.T) {
l := New(tc.l...)
assert.Equal(t, l, tc.exp)
})
}
}
func TestLabels_Merge(t *testing.T) {
cases := []struct {
n string
l, r Labels
exp Labels
}{
{
n: "no matching keys",
l: New("k05", "v05", "k03", "v03", "k01", "v01"),
r: New("k02", "v02", "k04", "v04", "k00", "v00"),
exp: New("k05", "v05", "k03", "v03", "k01", "v01", "k02", "v02", "k04", "v04", "k00", "v00"),
},
{
n: "multiple matching keys",
l: New("k05", "v05", "k03", "v03", "k01", "v01"),
r: New("k02", "v02", "k03", "v03a", "k05", "v05a"),
exp: New("k05", "v05a", "k03", "v03a", "k01", "v01", "k02", "v02"),
},
{
n: "source empty",
l: New(),
r: New("k02", "v02", "k04", "v04", "k00", "v00"),
exp: New("k02", "v02", "k04", "v04", "k00", "v00"),
},
{
n: "other empty",
l: New("k02", "v02", "k04", "v04", "k00", "v00"),
r: New(),
exp: New("k02", "v02", "k04", "v04", "k00", "v00"),
},
}
for _, tc := range cases {
t.Run(tc.n, func(t *testing.T) {
l := tc.l
l.Merge(tc.r)
assert.Equal(t, l, tc.exp)
})
}
}

18
pkg/tracing/rawspan.go Normal file
View File

@ -0,0 +1,18 @@
package tracing
import (
"time"
"github.com/influxdata/influxdb/v2/pkg/tracing/fields"
"github.com/influxdata/influxdb/v2/pkg/tracing/labels"
)
// RawSpan represents the data associated with a span.
type RawSpan struct {
Context SpanContext
ParentSpanID uint64 // ParentSpanID identifies the parent of this span or 0 if this is the root span.
Name string // Name is the operation name given to this span.
Start time.Time // Start identifies the start time of the span.
Labels labels.Labels // Labels contains additional metadata about this span.
Fields fields.Fields // Fields contains typed values associated with this span.
}

84
pkg/tracing/span.go Normal file
View File

@ -0,0 +1,84 @@
package tracing
import (
"sync"
"time"
"github.com/influxdata/influxdb/v2/pkg/tracing/fields"
"github.com/influxdata/influxdb/v2/pkg/tracing/labels"
)
// The Span type denotes a specific operation for a Trace.
// A Span may have one or more children, identifying additional
// details about a trace.
type Span struct {
tracer *Trace
mu sync.Mutex
raw RawSpan
}
type StartSpanOption interface {
applyStart(*Span)
}
// The StartTime start span option specifies the start time of
// the new span rather than using now.
type StartTime time.Time
func (t StartTime) applyStart(s *Span) {
s.raw.Start = time.Time(t)
}
// StartSpan creates a new child span using time.Now as the start time.
func (s *Span) StartSpan(name string, opt ...StartSpanOption) *Span {
return s.tracer.startSpan(name, s.raw.Context, opt)
}
// Context returns a SpanContext that can be serialized and passed to a remote node to continue a trace.
func (s *Span) Context() SpanContext {
return s.raw.Context
}
// SetLabels replaces any existing labels for the Span with args.
func (s *Span) SetLabels(args ...string) {
s.mu.Lock()
s.raw.Labels = labels.New(args...)
s.mu.Unlock()
}
// MergeLabels merges args with any existing labels defined
// for the Span.
func (s *Span) MergeLabels(args ...string) {
ls := labels.New(args...)
s.mu.Lock()
s.raw.Labels.Merge(ls)
s.mu.Unlock()
}
// SetFields replaces any existing fields for the Span with args.
func (s *Span) SetFields(set fields.Fields) {
s.mu.Lock()
s.raw.Fields = set
s.mu.Unlock()
}
// MergeFields merges the provides args with any existing fields defined
// for the Span.
func (s *Span) MergeFields(args ...fields.Field) {
set := fields.New(args...)
s.mu.Lock()
s.raw.Fields.Merge(set)
s.mu.Unlock()
}
// Finish marks the end of the span and records it to the associated Trace.
// If Finish is not called, the span will not appear in the trace.
func (s *Span) Finish() {
s.mu.Lock()
s.tracer.addRawSpan(s.raw)
s.mu.Unlock()
}
func (s *Span) Tree() *TreeNode {
return s.tracer.TreeFrom(s.raw.Context.SpanID)
}

27
pkg/tracing/spancontext.go Normal file
View File

@ -0,0 +1,27 @@
package tracing
import (
"github.com/gogo/protobuf/proto"
"github.com/influxdata/influxdb/v2/pkg/tracing/wire"
)
// A SpanContext represents the minimal information to identify a span in a trace.
// This is typically serialized to continue a trace on a remote node.
type SpanContext struct {
TraceID uint64 // TraceID is assigned a random number to this trace.
SpanID uint64 // SpanID is assigned a random number to identify this span.
}
func (s SpanContext) MarshalBinary() ([]byte, error) {
ws := wire.SpanContext(s)
return proto.Marshal(&ws)
}
func (s *SpanContext) UnmarshalBinary(data []byte) error {
var ws wire.SpanContext
err := proto.Unmarshal(data, &ws)
if err == nil {
*s = SpanContext(ws)
}
return err
}

138
pkg/tracing/trace.go Normal file
View File

@ -0,0 +1,138 @@
package tracing
import (
"sort"
"sync"
"time"
)
// The Trace type functions as a container for capturing Spans used to
// trace the execution of a request.
type Trace struct {
mu sync.Mutex
spans map[uint64]RawSpan
}
// NewTrace starts a new trace and returns a root span identified by the provided name.
//
// Additional options may be specified to override the default behavior when creating the span.
func NewTrace(name string, opt ...StartSpanOption) (*Trace, *Span) {
t := &Trace{spans: make(map[uint64]RawSpan)}
s := &Span{tracer: t}
s.raw.Name = name
s.raw.Context.TraceID, s.raw.Context.SpanID = randomID2()
setOptions(s, opt)
return t, s
}
// NewTraceFromSpan starts a new trace and returns the associated span, which is a child of the
// parent span context.
func NewTraceFromSpan(name string, parent SpanContext, opt ...StartSpanOption) (*Trace, *Span) {
t := &Trace{spans: make(map[uint64]RawSpan)}
s := &Span{tracer: t}
s.raw.Name = name
s.raw.ParentSpanID = parent.SpanID
s.raw.Context.TraceID = parent.TraceID
s.raw.Context.SpanID = randomID()
setOptions(s, opt)
return t, s
}
func (t *Trace) startSpan(name string, sc SpanContext, opt []StartSpanOption) *Span {
s := &Span{tracer: t}
s.raw.Name = name
s.raw.Context.SpanID = randomID()
s.raw.Context.TraceID = sc.TraceID
s.raw.ParentSpanID = sc.SpanID
setOptions(s, opt)
return s
}
func setOptions(s *Span, opt []StartSpanOption) {
for _, o := range opt {
o.applyStart(s)
}
if s.raw.Start.IsZero() {
s.raw.Start = time.Now()
}
}
func (t *Trace) addRawSpan(raw RawSpan) {
t.mu.Lock()
t.spans[raw.Context.SpanID] = raw
t.mu.Unlock()
}
// Tree returns a graph of the current trace.
func (t *Trace) Tree() *TreeNode {
t.mu.Lock()
defer t.mu.Unlock()
for _, s := range t.spans {
if s.ParentSpanID == 0 {
return t.treeFrom(s.Context.SpanID)
}
}
return nil
}
// Merge combines other with the current trace. This is
// typically necessary when traces are transferred from a remote.
func (t *Trace) Merge(other *Trace) {
for k, s := range other.spans {
t.spans[k] = s
}
}
func (t *Trace) TreeFrom(root uint64) *TreeNode {
t.mu.Lock()
defer t.mu.Unlock()
return t.treeFrom(root)
}
func (t *Trace) treeFrom(root uint64) *TreeNode {
c := map[uint64]*TreeNode{}
for k, s := range t.spans {
c[k] = &TreeNode{Raw: s}
}
if _, ok := c[root]; !ok {
return nil
}
for _, n := range c {
if n.Raw.ParentSpanID != 0 {
if pn := c[n.Raw.ParentSpanID]; pn != nil {
pn.Children = append(pn.Children, n)
}
}
}
// sort nodes
var v treeSortVisitor
Walk(&v, c[root])
return c[root]
}
type treeSortVisitor struct{}
func (v *treeSortVisitor) Visit(node *TreeNode) Visitor {
sort.Slice(node.Children, func(i, j int) bool {
lt, rt := node.Children[i].Raw.Start.UnixNano(), node.Children[j].Raw.Start.UnixNano()
if lt < rt {
return true
} else if lt > rt {
return false
}
ln, rn := node.Children[i].Raw.Name, node.Children[j].Raw.Name
return ln < rn
})
return v
}

136
pkg/tracing/trace_encoding.go Normal file
View File

@ -0,0 +1,136 @@
package tracing
import (
"math"
"time"
"github.com/gogo/protobuf/proto"
"github.com/influxdata/influxdb/v2/pkg/tracing/fields"
"github.com/influxdata/influxdb/v2/pkg/tracing/labels"
"github.com/influxdata/influxdb/v2/pkg/tracing/wire"
)
func fieldsToWire(set fields.Fields) []wire.Field {
var r []wire.Field
for _, f := range set {
wf := wire.Field{Key: f.Key()}
switch val := f.Value().(type) {
case string:
wf.FieldType = wire.FieldTypeString
wf.Value = &wire.Field_StringVal{StringVal: val}
case bool:
var numericVal int64
if val {
numericVal = 1
}
wf.FieldType = wire.FieldTypeBool
wf.Value = &wire.Field_NumericVal{NumericVal: numericVal}
case int64:
wf.FieldType = wire.FieldTypeInt64
wf.Value = &wire.Field_NumericVal{NumericVal: val}
case uint64:
wf.FieldType = wire.FieldTypeUint64
wf.Value = &wire.Field_NumericVal{NumericVal: int64(val)}
case time.Duration:
wf.FieldType = wire.FieldTypeDuration
wf.Value = &wire.Field_NumericVal{NumericVal: int64(val)}
case float64:
wf.FieldType = wire.FieldTypeFloat64
wf.Value = &wire.Field_NumericVal{NumericVal: int64(math.Float64bits(val))}
default:
continue
}
r = append(r, wf)
}
return r
}
func labelsToWire(set labels.Labels) []string {
var r []string
for i := range set {
r = append(r, set[i].Key, set[i].Value)
}
return r
}
func (t *Trace) MarshalBinary() ([]byte, error) {
wt := wire.Trace{}
for _, sp := range t.spans {
wt.Spans = append(wt.Spans, &wire.Span{
Context: wire.SpanContext{
TraceID: sp.Context.TraceID,
SpanID: sp.Context.SpanID,
},
ParentSpanID: sp.ParentSpanID,
Name: sp.Name,
Start: sp.Start,
Labels: labelsToWire(sp.Labels),
Fields: fieldsToWire(sp.Fields),
})
}
return proto.Marshal(&wt)
}
func wireToFields(wfs []wire.Field) fields.Fields {
var fs []fields.Field
for _, wf := range wfs {
switch wf.FieldType {
case wire.FieldTypeString:
fs = append(fs, fields.String(wf.Key, wf.GetStringVal()))
case wire.FieldTypeBool:
var boolVal bool
if wf.GetNumericVal() != 0 {
boolVal = true
}
fs = append(fs, fields.Bool(wf.Key, boolVal))
case wire.FieldTypeInt64:
fs = append(fs, fields.Int64(wf.Key, wf.GetNumericVal()))
case wire.FieldTypeUint64:
fs = append(fs, fields.Uint64(wf.Key, uint64(wf.GetNumericVal())))
case wire.FieldTypeDuration:
fs = append(fs, fields.Duration(wf.Key, time.Duration(wf.GetNumericVal())))
case wire.FieldTypeFloat64:
fs = append(fs, fields.Float64(wf.Key, math.Float64frombits(uint64(wf.GetNumericVal()))))
}
}
return fields.New(fs...)
}
func (t *Trace) UnmarshalBinary(data []byte) error {
var wt wire.Trace
if err := proto.Unmarshal(data, &wt); err != nil {
return err
}
t.spans = make(map[uint64]RawSpan)
for _, sp := range wt.Spans {
t.spans[sp.Context.SpanID] = RawSpan{
Context: SpanContext{
TraceID: sp.Context.TraceID,
SpanID: sp.Context.SpanID,
},
ParentSpanID: sp.ParentSpanID,
Name: sp.Name,
Start: sp.Start,
Labels: labels.New(sp.Labels...),
Fields: wireToFields(sp.Fields),
}
}
return nil
}

74
pkg/tracing/tree.go Normal file
View File

@ -0,0 +1,74 @@
package tracing
import (
"github.com/xlab/treeprint"
)
// A Visitor's Visit method is invoked for each node encountered by Walk.
// If the result of Visit is not nil, Walk visits each of the children.
type Visitor interface {
Visit(*TreeNode) Visitor
}
// A TreeNode represents a single node in the graph.
type TreeNode struct {
Raw RawSpan
Children []*TreeNode
}
// String returns the tree as a string.
func (t *TreeNode) String() string {
if t == nil {
return ""
}
tv := newTreeVisitor()
Walk(tv, t)
return tv.root.String()
}
// Walk traverses the graph in a depth-first order, calling v.Visit
// for each node until completion or v.Visit returns nil.
func Walk(v Visitor, node *TreeNode) {
if v = v.Visit(node); v == nil {
return
}
for _, c := range node.Children {
Walk(v, c)
}
}
type treeVisitor struct {
root treeprint.Tree
trees []treeprint.Tree
}
func newTreeVisitor() *treeVisitor {
t := treeprint.New()
return &treeVisitor{root: t, trees: []treeprint.Tree{t}}
}
func (v *treeVisitor) Visit(n *TreeNode) Visitor {
t := v.trees[len(v.trees)-1].AddBranch(n.Raw.Name)
v.trees = append(v.trees, t)
if labels := n.Raw.Labels; len(labels) > 0 {
l := t.AddBranch("labels")
for _, ll := range n.Raw.Labels {
l.AddNode(ll.Key + ": " + ll.Value)
}
}
for _, k := range n.Raw.Fields {
t.AddNode(k.String())
}
for _, cn := range n.Children {
Walk(v, cn)
}
v.trees[len(v.trees)-1] = nil
v.trees = v.trees[:len(v.trees)-1]
return nil
}

26
pkg/tracing/util.go Normal file
View File

@ -0,0 +1,26 @@
package tracing
import (
"math/rand"
"sync"
"time"
)
var (
seededIDGen = rand.New(rand.NewSource(time.Now().UnixNano()))
seededIDLock sync.Mutex
)
func randomID() (n uint64) {
seededIDLock.Lock()
n = uint64(seededIDGen.Int63())
seededIDLock.Unlock()
return
}
func randomID2() (n uint64, m uint64) {
seededIDLock.Lock()
n, m = uint64(seededIDGen.Int63()), uint64(seededIDGen.Int63())
seededIDLock.Unlock()
return
}

7
pkg/tracing/wire/binary.go Normal file
View File

@ -0,0 +1,7 @@
/*
Package wire is used to serialize a trace.
*/
package wire
//go:generate protoc -I$GOPATH/src -I. --gogofaster_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types:. binary.proto

1292
pkg/tracing/wire/binary.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

44
pkg/tracing/wire/binary.proto Normal file
View File

@ -0,0 +1,44 @@
syntax = "proto3";
package wire;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "google/protobuf/timestamp.proto";
message SpanContext {
uint64 trace_id = 1 [(gogoproto.customname) = "TraceID"];
uint64 span_id = 2 [(gogoproto.customname) = "SpanID"];
}
message Span {
SpanContext context = 1 [(gogoproto.nullable) = false];
uint64 parent_span_id = 2 [(gogoproto.customname) = "ParentSpanID"];
string name = 3;
google.protobuf.Timestamp start_time = 4 [(gogoproto.customname) = "Start", (gogoproto.stdtime) = true, (gogoproto.nullable) = false];
repeated string labels = 5;
repeated Field fields = 6 [(gogoproto.nullable) = false];
}
message Trace {
repeated Span spans = 1;
}
message Field {
enum FieldType {
option (gogoproto.goproto_enum_prefix) = false;
STRING = 0 [(gogoproto.enumvalue_customname) = "FieldTypeString"];
BOOL = 1 [(gogoproto.enumvalue_customname) = "FieldTypeBool"];
INT_64 = 2 [(gogoproto.enumvalue_customname) = "FieldTypeInt64"];
UINT_64 = 3 [(gogoproto.enumvalue_customname) = "FieldTypeUint64"];
DURATION = 4 [(gogoproto.enumvalue_customname) = "FieldTypeDuration"];
FLOAT_64 = 6 [(gogoproto.enumvalue_customname) = "FieldTypeFloat64"];
}
string key = 1;
FieldType field_type = 2 [(gogoproto.customname) = "FieldType"];
oneof value {
sfixed64 numeric_val = 3 [(gogoproto.customname) = "NumericVal"];
string string_val = 4 [(gogoproto.customname) = "StringVal"];
}
}

2
tsdb/cursors/cursor.go
View File

@ -53,6 +53,8 @@ type CursorIterator interface {
Stats() CursorStats
}
type CursorIterators []CursorIterator
// CursorStats represents stats collected by a cursor.
type CursorStats struct {
ScannedValues int // number of values scanned

63
v1/coordinator/config.go Normal file
View File

@ -0,0 +1,63 @@
// Package coordinator contains abstractions for writing points, executing statements,
// and accessing meta data.
package coordinator
import (
"time"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/toml"
"github.com/influxdata/influxdb/v2/v1/monitor/diagnostics"
)
const (
// DefaultWriteTimeout is the default timeout for a complete write to succeed.
DefaultWriteTimeout = 10 * time.Second
// DefaultMaxConcurrentQueries is the maximum number of running queries.
// A value of zero will make the maximum query limit unlimited.
DefaultMaxConcurrentQueries = 0
// DefaultMaxSelectPointN is the maximum number of points a SELECT can process.
// A value of zero will make the maximum point count unlimited.
DefaultMaxSelectPointN = 0
// DefaultMaxSelectSeriesN is the maximum number of series a SELECT can run.
// A value of zero will make the maximum series count unlimited.
DefaultMaxSelectSeriesN = 0
)
// Config represents the configuration for the coordinator service.
type Config struct {
WriteTimeout toml.Duration `toml:"write-timeout"`
MaxConcurrentQueries int `toml:"max-concurrent-queries"`
QueryTimeout toml.Duration `toml:"query-timeout"`
LogQueriesAfter toml.Duration `toml:"log-queries-after"`
MaxSelectPointN int `toml:"max-select-point"`
MaxSelectSeriesN int `toml:"max-select-series"`
MaxSelectBucketsN int `toml:"max-select-buckets"`
}
// NewConfig returns an instance of Config with defaults.
func NewConfig() Config {
return Config{
WriteTimeout: toml.Duration(DefaultWriteTimeout),
QueryTimeout: toml.Duration(query.DefaultQueryTimeout),
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
MaxSelectPointN: DefaultMaxSelectPointN,
MaxSelectSeriesN: DefaultMaxSelectSeriesN,
}
}
// Diagnostics returns a diagnostics representation of a subset of the Config.
func (c Config) Diagnostics() (*diagnostics.Diagnostics, error) {
return diagnostics.RowFromMap(map[string]interface{}{
"write-timeout": c.WriteTimeout,
"max-concurrent-queries": c.MaxConcurrentQueries,
"query-timeout": c.QueryTimeout,
"log-queries-after": c.LogQueriesAfter,
"max-select-point": c.MaxSelectPointN,
"max-select-series": c.MaxSelectSeriesN,
"max-select-buckets": c.MaxSelectBucketsN,
}), nil
}

24
v1/coordinator/config_test.go Normal file
View File

@ -0,0 +1,24 @@
package coordinator_test
import (
"testing"
"time"
"github.com/BurntSushi/toml"
"github.com/influxdata/influxdb/v2/v1/coordinator"
)
func TestConfig_Parse(t *testing.T) {
// Parse configuration.
var c coordinator.Config
if _, err := toml.Decode(`
write-timeout = "20s"
`, &c); err != nil {
t.Fatal(err)
}
// Validate configuration.
if time.Duration(c.WriteTimeout) != 20*time.Second {
t.Fatalf("unexpected write timeout s: %s", c.WriteTimeout)
}
}

36
v1/coordinator/meta_client.go Normal file
View File

@ -0,0 +1,36 @@
package coordinator
import (
"time"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxql"
)
// MetaClient is an interface for accessing meta data.
type MetaClient interface {
CreateContinuousQuery(database, name, query string) error
CreateDatabase(name string) (*meta.DatabaseInfo, error)
CreateDatabaseWithRetentionPolicy(name string, spec *meta.RetentionPolicySpec) (*meta.DatabaseInfo, error)
CreateRetentionPolicy(database string, spec *meta.RetentionPolicySpec, makeDefault bool) (*meta.RetentionPolicyInfo, error)
CreateSubscription(database, rp, name, mode string, destinations []string) error
CreateUser(name, password string, admin bool) (meta.User, error)
Database(name string) *meta.DatabaseInfo
Databases() []meta.DatabaseInfo
DropShard(id uint64) error
DropContinuousQuery(database, name string) error
DropDatabase(name string) error
DropRetentionPolicy(database, name string) error
DropSubscription(database, rp, name string) error
DropUser(name string) error
RetentionPolicy(database, name string) (rpi *meta.RetentionPolicyInfo, err error)
SetAdminPrivilege(username string, admin bool) error
SetPrivilege(username, database string, p influxql.Privilege) error
ShardGroupsByTimeRange(database, policy string, min, max time.Time) (a []meta.ShardGroupInfo, err error)
TruncateShardGroups(t time.Time) error
UpdateRetentionPolicy(database, name string, rpu *meta.RetentionPolicyUpdate, makeDefault bool) error
UpdateUser(name, password string) error
UserPrivilege(username, database string) (*influxql.Privilege, error)
UserPrivileges(username string) (map[string]influxql.Privilege, error)
Users() []meta.UserInfo
}

166
v1/coordinator/meta_client_test.go Normal file
View File

@ -0,0 +1,166 @@
package coordinator_test
import (
"time"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxql"
)
// MetaClient is a mockable implementation of cluster.MetaClient.
type MetaClient struct {
CreateContinuousQueryFn func(database, name, query string) error
CreateDatabaseFn func(name string) (*meta.DatabaseInfo, error)
CreateDatabaseWithRetentionPolicyFn func(name string, spec *meta.RetentionPolicySpec) (*meta.DatabaseInfo, error)
CreateRetentionPolicyFn func(database string, spec *meta.RetentionPolicySpec, makeDefault bool) (*meta.RetentionPolicyInfo, error)
CreateSubscriptionFn func(database, rp, name, mode string, destinations []string) error
CreateUserFn func(name, password string, admin bool) (meta.User, error)
DatabaseFn func(name string) *meta.DatabaseInfo
DatabasesFn func() []meta.DatabaseInfo
DataNodeFn func(id uint64) (*meta.NodeInfo, error)
DataNodesFn func() ([]meta.NodeInfo, error)
DeleteDataNodeFn func(id uint64) error
DeleteMetaNodeFn func(id uint64) error
DropContinuousQueryFn func(database, name string) error
DropDatabaseFn func(name string) error
DropRetentionPolicyFn func(database, name string) error
DropSubscriptionFn func(database, rp, name string) error
DropShardFn func(id uint64) error
DropUserFn func(name string) error
MetaNodesFn func() ([]meta.NodeInfo, error)
RetentionPolicyFn func(database, name string) (rpi *meta.RetentionPolicyInfo, err error)
SetAdminPrivilegeFn func(username string, admin bool) error
SetPrivilegeFn func(username, database string, p influxql.Privilege) error
ShardGroupsByTimeRangeFn func(database, policy string, min, max time.Time) (a []meta.ShardGroupInfo, err error)
TruncateShardGroupsFn func(t time.Time) error
UpdateRetentionPolicyFn func(database, name string, rpu *meta.RetentionPolicyUpdate, makeDefault bool) error
UpdateUserFn func(name, password string) error
UserPrivilegeFn func(username, database string) (*influxql.Privilege, error)
UserPrivilegesFn func(username string) (map[string]influxql.Privilege, error)
UsersFn func() []meta.UserInfo
}
func (c *MetaClient) CreateContinuousQuery(database, name, query string) error {
return c.CreateContinuousQueryFn(database, name, query)
}
func (c *MetaClient) CreateDatabase(name string) (*meta.DatabaseInfo, error) {
return c.CreateDatabaseFn(name)
}
func (c *MetaClient) CreateDatabaseWithRetentionPolicy(name string, spec *meta.RetentionPolicySpec) (*meta.DatabaseInfo, error) {
return c.CreateDatabaseWithRetentionPolicyFn(name, spec)
}
func (c *MetaClient) CreateRetentionPolicy(database string, spec *meta.RetentionPolicySpec, makeDefault bool) (*meta.RetentionPolicyInfo, error) {
return c.CreateRetentionPolicyFn(database, spec, makeDefault)
}
func (c *MetaClient) DropShard(id uint64) error {
return c.DropShardFn(id)
}
func (c *MetaClient) CreateSubscription(database, rp, name, mode string, destinations []string) error {
return c.CreateSubscriptionFn(database, rp, name, mode, destinations)
}
func (c *MetaClient) CreateUser(name, password string, admin bool) (meta.User, error) {
return c.CreateUserFn(name, password, admin)
}
func (c *MetaClient) Database(name string) *meta.DatabaseInfo {
return c.DatabaseFn(name)
}
func (c *MetaClient) Databases() []meta.DatabaseInfo {
return c.DatabasesFn()
}
func (c *MetaClient) DataNode(id uint64) (*meta.NodeInfo, error) {
return c.DataNodeFn(id)
}
func (c *MetaClient) DataNodes() ([]meta.NodeInfo, error) {
return c.DataNodesFn()
}
func (c *MetaClient) DeleteDataNode(id uint64) error {
return c.DeleteDataNodeFn(id)
}
func (c *MetaClient) DeleteMetaNode(id uint64) error {
return c.DeleteMetaNodeFn(id)
}
func (c *MetaClient) DropContinuousQuery(database, name string) error {
return c.DropContinuousQueryFn(database, name)
}
func (c *MetaClient) DropDatabase(name string) error {
return c.DropDatabaseFn(name)
}
func (c *MetaClient) DropRetentionPolicy(database, name string) error {
return c.DropRetentionPolicyFn(database, name)
}
func (c *MetaClient) DropSubscription(database, rp, name string) error {
return c.DropSubscriptionFn(database, rp, name)
}
func (c *MetaClient) DropUser(name string) error {
return c.DropUserFn(name)
}
func (c *MetaClient) MetaNodes() ([]meta.NodeInfo, error) {
return c.MetaNodesFn()
}
func (c *MetaClient) RetentionPolicy(database, name string) (rpi *meta.RetentionPolicyInfo, err error) {
return c.RetentionPolicyFn(database, name)
}
func (c *MetaClient) SetAdminPrivilege(username string, admin bool) error {
return c.SetAdminPrivilegeFn(username, admin)
}
func (c *MetaClient) SetPrivilege(username, database string, p influxql.Privilege) error {
return c.SetPrivilegeFn(username, database, p)
}
func (c *MetaClient) ShardGroupsByTimeRange(database, policy string, min, max time.Time) (a []meta.ShardGroupInfo, err error) {
return c.ShardGroupsByTimeRangeFn(database, policy, min, max)
}
func (c *MetaClient) TruncateShardGroups(t time.Time) error {
return c.TruncateShardGroupsFn(t)
}
func (c *MetaClient) UpdateRetentionPolicy(database, name string, rpu *meta.RetentionPolicyUpdate, makeDefault bool) error {
return c.UpdateRetentionPolicyFn(database, name, rpu, makeDefault)
}
func (c *MetaClient) UpdateUser(name, password string) error {
return c.UpdateUserFn(name, password)
}
func (c *MetaClient) UserPrivilege(username, database string) (*influxql.Privilege, error) {
return c.UserPrivilegeFn(username, database)
}
func (c *MetaClient) UserPrivileges(username string) (map[string]influxql.Privilege, error) {
return c.UserPrivilegesFn(username)
}
func (c *MetaClient) Users() []meta.UserInfo {
return c.UsersFn()
}
// DefaultMetaClientDatabaseFn returns a single database (db0) with a retention policy.
func DefaultMetaClientDatabaseFn(name string) *meta.DatabaseInfo {
return &meta.DatabaseInfo{
Name: DefaultDatabase,
DefaultRetentionPolicy: DefaultRetentionPolicy,
}
}

398
v1/coordinator/points_writer.go Normal file
View File

@ -0,0 +1,398 @@
package coordinator
import (
"errors"
"fmt"
"sort"
"sync"
"sync/atomic"
"time"
influxdb "github.com/influxdata/influxdb/v2/v1"
"github.com/influxdata/influxdb/v2/v1/models"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxdb/v2/v1/tsdb"
"go.uber.org/zap"
)
// The keys for statistics generated by the "write" module.
const (
statWriteReq = "req"
statPointWriteReq = "pointReq"
statPointWriteReqLocal = "pointReqLocal"
statWriteOK = "writeOk"
statWriteDrop = "writeDrop"
statWriteTimeout = "writeTimeout"
statWriteErr = "writeError"
statSubWriteOK = "subWriteOk"
statSubWriteDrop = "subWriteDrop"
)
var (
// ErrTimeout is returned when a write times out.
ErrTimeout = errors.New("timeout")
// ErrPartialWrite is returned when a write partially succeeds but does
// not meet the requested consistency level.
ErrPartialWrite = errors.New("partial write")
// ErrWriteFailed is returned when no writes succeeded.
ErrWriteFailed = errors.New("write failed")
)
// PointsWriter handles writes across multiple local and remote data nodes.
type PointsWriter struct {
mu sync.RWMutex
closing chan struct{}
WriteTimeout time.Duration
Logger *zap.Logger
Node *influxdb.Node
MetaClient interface {
Database(name string) (di *meta.DatabaseInfo)
RetentionPolicy(database, policy string) (*meta.RetentionPolicyInfo, error)
CreateShardGroup(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
}
TSDBStore interface {
CreateShard(database, retentionPolicy string, shardID uint64, enabled bool) error
WriteToShard(shardID uint64, points []models.Point) error
}
subPoints []chan<- *WritePointsRequest
stats *WriteStatistics
}
// WritePointsRequest represents a request to write point data to the cluster.
type WritePointsRequest struct {
Database string
RetentionPolicy string
Points []models.Point
}
// AddPoint adds a point to the WritePointRequest with field key 'value'
func (w *WritePointsRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string) {
pt, err := models.NewPoint(
name, models.NewTags(tags), map[string]interface{}{"value": value}, timestamp,
)
if err != nil {
return
}
w.Points = append(w.Points, pt)
}
// NewPointsWriter returns a new instance of PointsWriter for a node.
func NewPointsWriter() *PointsWriter {
return &PointsWriter{
closing: make(chan struct{}),
WriteTimeout: DefaultWriteTimeout,
Logger: zap.NewNop(),
stats: &WriteStatistics{},
}
}
// ShardMapping contains a mapping of shards to points.
type ShardMapping struct {
n int
Points map[uint64][]models.Point // The points associated with a shard ID
Shards map[uint64]*meta.ShardInfo // The shards that have been mapped, keyed by shard ID
Dropped []models.Point // Points that were dropped
}
// NewShardMapping creates an empty ShardMapping.
func NewShardMapping(n int) *ShardMapping {
return &ShardMapping{
n: n,
Points: map[uint64][]models.Point{},
Shards: map[uint64]*meta.ShardInfo{},
}
}
// MapPoint adds the point to the ShardMapping, associated with the given shardInfo.
func (s *ShardMapping) MapPoint(shardInfo *meta.ShardInfo, p models.Point) {
if cap(s.Points[shardInfo.ID]) < s.n {
s.Points[shardInfo.ID] = make([]models.Point, 0, s.n)
}
s.Points[shardInfo.ID] = append(s.Points[shardInfo.ID], p)
s.Shards[shardInfo.ID] = shardInfo
}
// Open opens the communication channel with the point writer.
func (w *PointsWriter) Open() error {
w.mu.Lock()
defer w.mu.Unlock()
w.closing = make(chan struct{})
return nil
}
// Close closes the communication channel with the point writer.
func (w *PointsWriter) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closing != nil {
close(w.closing)
}
if w.subPoints != nil {
// 'nil' channels always block so this makes the
// select statement in WritePoints hit its default case
// dropping any in-flight writes.
w.subPoints = nil
}
return nil
}
func (w *PointsWriter) AddWriteSubscriber(c chan<- *WritePointsRequest) {
w.subPoints = append(w.subPoints, c)
}
// WithLogger sets the Logger on w.
func (w *PointsWriter) WithLogger(log *zap.Logger) {
w.Logger = log.With(zap.String("service", "write"))
}
// WriteStatistics keeps statistics related to the PointsWriter.
type WriteStatistics struct {
WriteReq int64
PointWriteReq int64
PointWriteReqLocal int64
WriteOK int64
WriteDropped int64
WriteTimeout int64
WriteErr int64
SubWriteOK int64
SubWriteDrop int64
}
// Statistics returns statistics for periodic monitoring.
func (w *PointsWriter) Statistics(tags map[string]string) []models.Statistic {
return []models.Statistic{{
Name: "write",
Tags: tags,
Values: map[string]interface{}{
statWriteReq: atomic.LoadInt64(&w.stats.WriteReq),
statPointWriteReq: atomic.LoadInt64(&w.stats.PointWriteReq),
statPointWriteReqLocal: atomic.LoadInt64(&w.stats.PointWriteReqLocal),
statWriteOK: atomic.LoadInt64(&w.stats.WriteOK),
statWriteDrop: atomic.LoadInt64(&w.stats.WriteDropped),
statWriteTimeout: atomic.LoadInt64(&w.stats.WriteTimeout),
statWriteErr: atomic.LoadInt64(&w.stats.WriteErr),
statSubWriteOK: atomic.LoadInt64(&w.stats.SubWriteOK),
statSubWriteDrop: atomic.LoadInt64(&w.stats.SubWriteDrop),
},
}}
}
// MapShards maps the points contained in wp to a ShardMapping. If a point
// maps to a shard group or shard that does not currently exist, it will be
// created before returning the mapping.
func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error) {
rp, err := w.MetaClient.RetentionPolicy(wp.Database, wp.RetentionPolicy)
if err != nil {
return nil, err
} else if rp == nil {
return nil, influxdb.ErrRetentionPolicyNotFound(wp.RetentionPolicy)
}
// Holds all the shard groups and shards that are required for writes.
list := make(sgList, 0, 8)
min := time.Unix(0, models.MinNanoTime)
if rp.Duration > 0 {
min = time.Now().Add(-rp.Duration)
}
for _, p := range wp.Points {
// Either the point is outside the scope of the RP, or we already have
// a suitable shard group for the point.
if p.Time().Before(min) || list.Covers(p.Time()) {
continue
}
// No shard groups overlap with the point's time, so we will create
// a new shard group for this point.
sg, err := w.MetaClient.CreateShardGroup(wp.Database, wp.RetentionPolicy, p.Time())
if err != nil {
return nil, err
}
if sg == nil {
return nil, errors.New("nil shard group")
}
list = list.Append(*sg)
}
mapping := NewShardMapping(len(wp.Points))
for _, p := range wp.Points {
sg := list.ShardGroupAt(p.Time())
if sg == nil {
// We didn't create a shard group because the point was outside the
// scope of the RP.
mapping.Dropped = append(mapping.Dropped, p)
atomic.AddInt64(&w.stats.WriteDropped, 1)
continue
}
sh := sg.ShardFor(p.HashID())
mapping.MapPoint(&sh, p)
}
return mapping, nil
}
// sgList is a wrapper around a meta.ShardGroupInfos where we can also check
// if a given time is covered by any of the shard groups in the list.
type sgList meta.ShardGroupInfos
func (l sgList) Covers(t time.Time) bool {
if len(l) == 0 {
return false
}
return l.ShardGroupAt(t) != nil
}
// ShardGroupAt attempts to find a shard group that could contain a point
// at the given time.
//
// Shard groups are sorted first according to end time, and then according
// to start time. Therefore, if there are multiple shard groups that match
// this point's time they will be preferred in this order:
//
// - a shard group with the earliest end time;
// - (assuming identical end times) the shard group with the earliest start time.
func (l sgList) ShardGroupAt(t time.Time) *meta.ShardGroupInfo {
idx := sort.Search(len(l), func(i int) bool { return l[i].EndTime.After(t) })
// We couldn't find a shard group the point falls into.
if idx == len(l) || t.Before(l[idx].StartTime) {
return nil
}
return &l[idx]
}
// Append appends a shard group to the list, and returns a sorted list.
func (l sgList) Append(sgi meta.ShardGroupInfo) sgList {
next := append(l, sgi)
sort.Sort(meta.ShardGroupInfos(next))
return next
}
// WritePointsInto is a copy of WritePoints that uses a tsdb structure instead of
// a cluster structure for information. This is to avoid a circular dependency.
func (w *PointsWriter) WritePointsInto(p *IntoWriteRequest) error {
return w.WritePointsPrivileged(p.Database, p.RetentionPolicy, models.ConsistencyLevelOne, p.Points)
}
// WritePoints writes the data to the underlying storage. consitencyLevel and user are only used for clustered scenarios
func (w *PointsWriter) WritePoints(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, user meta.User, points []models.Point) error {
return w.WritePointsPrivileged(database, retentionPolicy, consistencyLevel, points)
}
// WritePointsPrivileged writes the data to the underlying storage, consitencyLevel is only used for clustered scenarios
func (w *PointsWriter) WritePointsPrivileged(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error {
atomic.AddInt64(&w.stats.WriteReq, 1)
atomic.AddInt64(&w.stats.PointWriteReq, int64(len(points)))
if retentionPolicy == "" {
db := w.MetaClient.Database(database)
if db == nil {
return influxdb.ErrDatabaseNotFound(database)
}
retentionPolicy = db.DefaultRetentionPolicy
}
shardMappings, err := w.MapShards(&WritePointsRequest{Database: database, RetentionPolicy: retentionPolicy, Points: points})
if err != nil {
return err
}
// Write each shard in it's own goroutine and return as soon as one fails.
ch := make(chan error, len(shardMappings.Points))
for shardID, points := range shardMappings.Points {
go func(shard *meta.ShardInfo, database, retentionPolicy string, points []models.Point) {
err := w.writeToShard(shard, database, retentionPolicy, points)
if err == tsdb.ErrShardDeletion {
err = tsdb.PartialWriteError{Reason: fmt.Sprintf("shard %d is pending deletion", shard.ID), Dropped: len(points)}
}
ch <- err
}(shardMappings.Shards[shardID], database, retentionPolicy, points)
}
// Send points to subscriptions if possible.
var ok, dropped int64
pts := &WritePointsRequest{Database: database, RetentionPolicy: retentionPolicy, Points: points}
// We need to lock just in case the channel is about to be nil'ed
w.mu.RLock()
for _, ch := range w.subPoints {
select {
case ch <- pts:
ok++
default:
dropped++
}
}
w.mu.RUnlock()
if ok > 0 {
atomic.AddInt64(&w.stats.SubWriteOK, ok)
}
if dropped > 0 {
atomic.AddInt64(&w.stats.SubWriteDrop, dropped)
}
if err == nil && len(shardMappings.Dropped) > 0 {
err = tsdb.PartialWriteError{Reason: "points beyond retention policy", Dropped: len(shardMappings.Dropped)}
}
timeout := time.NewTimer(w.WriteTimeout)
defer timeout.Stop()
for range shardMappings.Points {
select {
case <-w.closing:
return ErrWriteFailed
case <-timeout.C:
atomic.AddInt64(&w.stats.WriteTimeout, 1)
// return timeout error to caller
return ErrTimeout
case err := <-ch:
if err != nil {
return err
}
}
}
return err
}
// writeToShards writes points to a shard.
func (w *PointsWriter) writeToShard(shard *meta.ShardInfo, database, retentionPolicy string, points []models.Point) error {
atomic.AddInt64(&w.stats.PointWriteReqLocal, int64(len(points)))
err := w.TSDBStore.WriteToShard(shard.ID, points)
if err == nil {
atomic.AddInt64(&w.stats.WriteOK, 1)
return nil
}
// Except tsdb.ErrShardNotFound no error can be handled here
if err != tsdb.ErrShardNotFound {
atomic.AddInt64(&w.stats.WriteErr, 1)
return err
}
// If we've written to shard that should exist on the current node, but the store has
// not actually created this shard, tell it to create it and retry the write
if err = w.TSDBStore.CreateShard(database, retentionPolicy, shard.ID, true); err != nil {
w.Logger.Info("Write failed", zap.Uint64("shard", shard.ID), zap.Error(err))
atomic.AddInt64(&w.stats.WriteErr, 1)
return err
}
if err = w.TSDBStore.WriteToShard(shard.ID, points); err != nil {
w.Logger.Info("Write failed", zap.Uint64("shard", shard.ID), zap.Error(err))
atomic.AddInt64(&w.stats.WriteErr, 1)
return err
}
atomic.AddInt64(&w.stats.WriteOK, 1)
return nil
}

46
v1/coordinator/points_writer_internal_test.go Normal file
View File

@ -0,0 +1,46 @@
package coordinator
import (
"testing"
"time"
)
func TestSgList_ShardGroupAt(t *testing.T) {
base := time.Date(2016, 10, 19, 0, 0, 0, 0, time.UTC)
day := func(n int) time.Time {
return base.Add(time.Duration(24*n) * time.Hour)
}
list := sgList{
{ID: 1, StartTime: day(0), EndTime: day(1)},
{ID: 2, StartTime: day(1), EndTime: day(2)},
{ID: 3, StartTime: day(2), EndTime: day(3)},
// SG day 3 to day 4 missing...
{ID: 4, StartTime: day(4), EndTime: day(5)},
{ID: 5, StartTime: day(5), EndTime: day(6)},
}
examples := []struct {
T time.Time
ShardGroupID uint64 // 0 will indicate we don't expect a shard group
}{
{T: base.Add(-time.Minute), ShardGroupID: 0}, // Before any SG
{T: day(0), ShardGroupID: 1},
{T: day(0).Add(time.Minute), ShardGroupID: 1},
{T: day(1), ShardGroupID: 2},
{T: day(3).Add(time.Minute), ShardGroupID: 0}, // No matching SG
{T: day(5).Add(time.Hour), ShardGroupID: 5},
}
for i, example := range examples {
sg := list.ShardGroupAt(example.T)
var id uint64
if sg != nil {
id = sg.ID
}
if got, exp := id, example.ShardGroupID; got != exp {
t.Errorf("[Example %d] got %v, expected %v", i+1, got, exp)
}
}
}

683
v1/coordinator/points_writer_test.go Normal file
View File

@ -0,0 +1,683 @@
package coordinator_test
import (
"fmt"
"reflect"
"sync"
"sync/atomic"
"testing"
"time"
influxdb "github.com/influxdata/influxdb/v2/v1"
"github.com/influxdata/influxdb/v2/v1/coordinator"
"github.com/influxdata/influxdb/v2/v1/models"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxdb/v2/v1/tsdb"
)
// TODO(benbjohnson): Rewrite tests to use cluster_test.MetaClient.
// Ensures the points writer maps a single point to a single shard.
func TestPointsWriter_MapShards_One(t *testing.T) {
ms := PointsWriterMetaClient{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
ms.NodeIDFn = func() uint64 { return 1 }
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
return &rp.ShardGroups[0], nil
}
c := coordinator.PointsWriter{MetaClient: ms}
pr := &coordinator.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
}
pr.AddPoint("cpu", 1.0, time.Now(), nil)
var (
shardMappings *coordinator.ShardMapping
err error
)
if shardMappings, err = c.MapShards(pr); err != nil {
t.Fatalf("unexpected an error: %v", err)
}
if exp := 1; len(shardMappings.Points) != exp {
t.Errorf("MapShards() len mismatch. got %v, exp %v", len(shardMappings.Points), exp)
}
}
// Ensures the points writer maps to a new shard group when the shard duration
// is changed.
func TestPointsWriter_MapShards_AlterShardDuration(t *testing.T) {
ms := PointsWriterMetaClient{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
ms.NodeIDFn = func() uint64 { return 1 }
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
var (
i int
now = time.Now()
)
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
sg := []meta.ShardGroupInfo{
meta.ShardGroupInfo{
Shards: make([]meta.ShardInfo, 1),
StartTime: now, EndTime: now.Add(rp.Duration).Add(-1),
},
meta.ShardGroupInfo{
Shards: make([]meta.ShardInfo, 1),
StartTime: now.Add(time.Hour), EndTime: now.Add(3 * time.Hour).Add(rp.Duration).Add(-1),
},
}[i]
i++
return &sg, nil
}
c := coordinator.NewPointsWriter()
c.MetaClient = ms
pr := &coordinator.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
}
pr.AddPoint("cpu", 1.0, now, nil)
pr.AddPoint("cpu", 2.0, now.Add(2*time.Second), nil)
var (
shardMappings *coordinator.ShardMapping
err error
)
if shardMappings, err = c.MapShards(pr); err != nil {
t.Fatalf("unexpected an error: %v", err)
}
if got, exp := len(shardMappings.Points[0]), 2; got != exp {
t.Fatalf("got %d point(s), expected %d", got, exp)
}
if got, exp := len(shardMappings.Shards), 1; got != exp {
t.Errorf("got %d shard(s), expected %d", got, exp)
}
// Now we alter the retention policy duration.
rp.ShardGroupDuration = 3 * time.Hour
pr = &coordinator.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
}
pr.AddPoint("cpu", 1.0, now.Add(2*time.Hour), nil)
// Point is beyond previous shard group so a new shard group should be
// created.
if _, err = c.MapShards(pr); err != nil {
t.Fatalf("unexpected an error: %v", err)
}
// We can check value of i since it's only incremeneted when a shard group
// is created.
if got, exp := i, 2; got != exp {
t.Fatal("new shard group was not created, expected it to be")
}
}
// Ensures the points writer maps a multiple points across shard group boundaries.
func TestPointsWriter_MapShards_Multiple(t *testing.T) {
ms := PointsWriterMetaClient{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
rp.ShardGroupDuration = time.Hour
AttachShardGroupInfo(rp, []meta.ShardOwner{
{NodeID: 1},
{NodeID: 2},
{NodeID: 3},
})
AttachShardGroupInfo(rp, []meta.ShardOwner{
{NodeID: 1},
{NodeID: 2},
{NodeID: 3},
})
ms.NodeIDFn = func() uint64 { return 1 }
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
for i, sg := range rp.ShardGroups {
if timestamp.Equal(sg.StartTime) || timestamp.After(sg.StartTime) && timestamp.Before(sg.EndTime) {
return &rp.ShardGroups[i], nil
}
}
panic("should not get here")
}
c := coordinator.NewPointsWriter()
c.MetaClient = ms
defer c.Close()
pr := &coordinator.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
}
// Three points that range over the shardGroup duration (1h) and should map to two
// distinct shards
pr.AddPoint("cpu", 1.0, time.Now(), nil)
pr.AddPoint("cpu", 2.0, time.Now().Add(time.Hour), nil)
pr.AddPoint("cpu", 3.0, time.Now().Add(time.Hour+time.Second), nil)
var (
shardMappings *coordinator.ShardMapping
err error
)
if shardMappings, err = c.MapShards(pr); err != nil {
t.Fatalf("unexpected an error: %v", err)
}
if exp := 2; len(shardMappings.Points) != exp {
t.Errorf("MapShards() len mismatch. got %v, exp %v", len(shardMappings.Points), exp)
}
for _, points := range shardMappings.Points {
// First shard should have 1 point w/ first point added
if len(points) == 1 && points[0].Time() != pr.Points[0].Time() {
t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[0].Time(), pr.Points[0].Time())
}
// Second shard should have the last two points added
if len(points) == 2 && points[0].Time() != pr.Points[1].Time() {
t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[0].Time(), pr.Points[1].Time())
}
if len(points) == 2 && points[1].Time() != pr.Points[2].Time() {
t.Fatalf("MapShards() value mismatch. got %v, exp %v", points[1].Time(), pr.Points[2].Time())
}
}
}
// Ensures the points writer does not map points beyond the retention policy.
func TestPointsWriter_MapShards_Invalid(t *testing.T) {
ms := PointsWriterMetaClient{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
return &rp.ShardGroups[0], nil
}
c := coordinator.NewPointsWriter()
c.MetaClient = ms
defer c.Close()
pr := &coordinator.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
}
// Add a point that goes beyond the current retention policy.
pr.AddPoint("cpu", 1.0, time.Now().Add(-2*time.Hour), nil)
var (
shardMappings *coordinator.ShardMapping
err error
)
if shardMappings, err = c.MapShards(pr); err != nil {
t.Fatalf("unexpected an error: %v", err)
}
if got, exp := len(shardMappings.Points), 0; got != exp {
t.Errorf("MapShards() len mismatch. got %v, exp %v", got, exp)
}
if got, exp := len(shardMappings.Dropped), 1; got != exp {
t.Fatalf("MapShard() dropped mismatch: got %v, exp %v", got, exp)
}
}
func TestPointsWriter_WritePoints(t *testing.T) {
tests := []struct {
name string
database string
retentionPolicy string
// the responses returned by each shard write call. node ID 1 = pos 0
err []error
expErr error
}{
{
name: "write one success",
database: "mydb",
retentionPolicy: "myrp",
err: []error{nil, nil, nil},
expErr: nil,
},
// Write to non-existent database
{
name: "write to non-existent database",
database: "doesnt_exist",
retentionPolicy: "",
err: []error{nil, nil, nil},
expErr: fmt.Errorf("database not found: doesnt_exist"),
},
}
for _, test := range tests {
pr := &coordinator.WritePointsRequest{
Database: test.database,
RetentionPolicy: test.retentionPolicy,
}
// Ensure that the test shard groups are created before the points
// are created.
ms := NewPointsWriterMetaClient()
// Three points that range over the shardGroup duration (1h) and should map to two
// distinct shards
pr.AddPoint("cpu", 1.0, time.Now(), nil)
pr.AddPoint("cpu", 2.0, time.Now().Add(time.Hour), nil)
pr.AddPoint("cpu", 3.0, time.Now().Add(time.Hour+time.Second), nil)
// copy to prevent data race
theTest := test
sm := coordinator.NewShardMapping(16)
sm.MapPoint(
&meta.ShardInfo{ID: uint64(1), Owners: []meta.ShardOwner{
{NodeID: 1},
{NodeID: 2},
{NodeID: 3},
}},
pr.Points[0])
sm.MapPoint(
&meta.ShardInfo{ID: uint64(2), Owners: []meta.ShardOwner{
{NodeID: 1},
{NodeID: 2},
{NodeID: 3},
}},
pr.Points[1])
sm.MapPoint(
&meta.ShardInfo{ID: uint64(2), Owners: []meta.ShardOwner{
{NodeID: 1},
{NodeID: 2},
{NodeID: 3},
}},
pr.Points[2])
// Local coordinator.Node ShardWriter
// lock on the write increment since these functions get called in parallel
var mu sync.Mutex
store := &fakeStore{
WriteFn: func(shardID uint64, points []models.Point) error {
mu.Lock()
defer mu.Unlock()
return theTest.err[0]
},
}
ms.DatabaseFn = func(database string) *meta.DatabaseInfo {
return nil
}
ms.NodeIDFn = func() uint64 { return 1 }
subPoints := make(chan *coordinator.WritePointsRequest, 1)
sub := Subscriber{}
sub.PointsFn = func() chan<- *coordinator.WritePointsRequest {
return subPoints
}
c := coordinator.NewPointsWriter()
c.MetaClient = ms
c.TSDBStore = store
c.AddWriteSubscriber(sub.Points())
c.Node = &influxdb.Node{ID: 1}
c.Open()
defer c.Close()
err := c.WritePointsPrivileged(pr.Database, pr.RetentionPolicy, models.ConsistencyLevelOne, pr.Points)
if err == nil && test.expErr != nil {
t.Errorf("PointsWriter.WritePointsPrivileged(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
}
if err != nil && test.expErr == nil {
t.Errorf("PointsWriter.WritePointsPrivileged(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
}
if err != nil && test.expErr != nil && err.Error() != test.expErr.Error() {
t.Errorf("PointsWriter.WritePointsPrivileged(): '%s' error: got %v, exp %v", test.name, err, test.expErr)
}
if test.expErr == nil {
select {
case p := <-subPoints:
if !reflect.DeepEqual(p, pr) {
t.Errorf("PointsWriter.WritePointsPrivileged(): '%s' error: unexpected WritePointsRequest got %v, exp %v", test.name, p, pr)
}
default:
t.Errorf("PointsWriter.WritePointsPrivileged(): '%s' error: Subscriber.Points not called", test.name)
}
}
}
}
func TestPointsWriter_WritePoints_Dropped(t *testing.T) {
pr := &coordinator.WritePointsRequest{
Database: "mydb",
RetentionPolicy: "myrp",
}
// Ensure that the test shard groups are created before the points
// are created.
ms := NewPointsWriterMetaClient()
// Three points that range over the shardGroup duration (1h) and should map to two
// distinct shards
pr.AddPoint("cpu", 1.0, time.Now().Add(-24*time.Hour), nil)
// copy to prevent data race
sm := coordinator.NewShardMapping(16)
// ShardMapper dropped this point
sm.Dropped = append(sm.Dropped, pr.Points[0])
// Local coordinator.Node ShardWriter
// lock on the write increment since these functions get called in parallel
var mu sync.Mutex
store := &fakeStore{
WriteFn: func(shardID uint64, points []models.Point) error {
mu.Lock()
defer mu.Unlock()
return nil
},
}
ms.DatabaseFn = func(database string) *meta.DatabaseInfo {
return nil
}
ms.NodeIDFn = func() uint64 { return 1 }
subPoints := make(chan *coordinator.WritePointsRequest, 1)
sub := Subscriber{}
sub.PointsFn = func() chan<- *coordinator.WritePointsRequest {
return subPoints
}
c := coordinator.NewPointsWriter()
c.MetaClient = ms
c.TSDBStore = store
c.AddWriteSubscriber(sub.Points())
c.Node = &influxdb.Node{ID: 1}
c.Open()
defer c.Close()
err := c.WritePointsPrivileged(pr.Database, pr.RetentionPolicy, models.ConsistencyLevelOne, pr.Points)
if _, ok := err.(tsdb.PartialWriteError); !ok {
t.Errorf("PointsWriter.WritePoints(): got %v, exp %v", err, tsdb.PartialWriteError{})
}
}
type fakePointsWriter struct {
WritePointsIntoFn func(*coordinator.IntoWriteRequest) error
}
func (f *fakePointsWriter) WritePointsInto(req *coordinator.IntoWriteRequest) error {
return f.WritePointsIntoFn(req)
}
func TestBufferedPointsWriter(t *testing.T) {
db := "db0"
rp := "rp0"
capacity := 10000
writePointsIntoCnt := 0
pointsWritten := []models.Point{}
reset := func() {
writePointsIntoCnt = 0
pointsWritten = pointsWritten[:0]
}
fakeWriter := &fakePointsWriter{
WritePointsIntoFn: func(req *coordinator.IntoWriteRequest) error {
writePointsIntoCnt++
pointsWritten = append(pointsWritten, req.Points...)
return nil
},
}
w := coordinator.NewBufferedPointsWriter(fakeWriter, db, rp, capacity)
// Test that capacity and length are correct for new buffered writer.
if w.Cap() != capacity {
t.Fatalf("exp %d, got %d", capacity, w.Cap())
} else if w.Len() != 0 {
t.Fatalf("exp %d, got %d", 0, w.Len())
}
// Test flushing an empty buffer.
if err := w.Flush(); err != nil {
t.Fatal(err)
} else if writePointsIntoCnt > 0 {
t.Fatalf("exp 0, got %d", writePointsIntoCnt)
}
// Test writing zero points.
if err := w.WritePointsInto(&coordinator.IntoWriteRequest{
Database: db,
RetentionPolicy: rp,
Points: []models.Point{},
}); err != nil {
t.Fatal(err)
} else if writePointsIntoCnt > 0 {
t.Fatalf("exp 0, got %d", writePointsIntoCnt)
} else if w.Len() > 0 {
t.Fatalf("exp 0, got %d", w.Len())
}
// Test writing single large bunch of points points.
req := coordinator.WritePointsRequest{
Database: db,
RetentionPolicy: rp,
}
numPoints := int(float64(capacity) * 5.5)
for i := 0; i < numPoints; i++ {
req.AddPoint("cpu", float64(i), time.Now().Add(time.Duration(i)*time.Second), nil)
}
r := coordinator.IntoWriteRequest(req)
if err := w.WritePointsInto(&r); err != nil {
t.Fatal(err)
} else if writePointsIntoCnt != 5 {
t.Fatalf("exp 5, got %d", writePointsIntoCnt)
} else if w.Len() != capacity/2 {
t.Fatalf("exp %d, got %d", capacity/2, w.Len())
} else if len(pointsWritten) != numPoints-capacity/2 {
t.Fatalf("exp %d, got %d", numPoints-capacity/2, len(pointsWritten))
}
if err := w.Flush(); err != nil {
t.Fatal(err)
} else if writePointsIntoCnt != 6 {
t.Fatalf("exp 6, got %d", writePointsIntoCnt)
} else if w.Len() != 0 {
t.Fatalf("exp 0, got %d", w.Len())
} else if len(pointsWritten) != numPoints {
t.Fatalf("exp %d, got %d", numPoints, len(pointsWritten))
} else if !reflect.DeepEqual(r.Points, pointsWritten) {
t.Fatal("points don't match")
}
reset()
// Test writing points one at a time.
for i := range r.Points {
if err := w.WritePointsInto(&coordinator.IntoWriteRequest{
Database: db,
RetentionPolicy: rp,
Points: r.Points[i : i+1],
}); err != nil {
t.Fatal(err)
}
}
if err := w.Flush(); err != nil {
t.Fatal(err)
} else if writePointsIntoCnt != 6 {
t.Fatalf("exp 6, got %d", writePointsIntoCnt)
} else if w.Len() != 0 {
t.Fatalf("exp 0, got %d", w.Len())
} else if len(pointsWritten) != numPoints {
t.Fatalf("exp %d, got %d", numPoints, len(pointsWritten))
} else if !reflect.DeepEqual(r.Points, pointsWritten) {
t.Fatal("points don't match")
}
}
var shardID uint64
type fakeStore struct {
WriteFn func(shardID uint64, points []models.Point) error
CreateShardfn func(database, retentionPolicy string, shardID uint64, enabled bool) error
}
func (f *fakeStore) WriteToShard(shardID uint64, points []models.Point) error {
return f.WriteFn(shardID, points)
}
func (f *fakeStore) CreateShard(database, retentionPolicy string, shardID uint64, enabled bool) error {
return f.CreateShardfn(database, retentionPolicy, shardID, enabled)
}
func NewPointsWriterMetaClient() *PointsWriterMetaClient {
ms := &PointsWriterMetaClient{}
rp := NewRetentionPolicy("myp", time.Hour, 3)
AttachShardGroupInfo(rp, []meta.ShardOwner{
{NodeID: 1},
{NodeID: 2},
{NodeID: 3},
})
AttachShardGroupInfo(rp, []meta.ShardOwner{
{NodeID: 1},
{NodeID: 2},
{NodeID: 3},
})
ms.RetentionPolicyFn = func(db, retentionPolicy string) (*meta.RetentionPolicyInfo, error) {
return rp, nil
}
ms.CreateShardGroupIfNotExistsFn = func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
for i, sg := range rp.ShardGroups {
if timestamp.Equal(sg.StartTime) || timestamp.After(sg.StartTime) && timestamp.Before(sg.EndTime) {
return &rp.ShardGroups[i], nil
}
}
panic("should not get here")
}
return ms
}
type PointsWriterMetaClient struct {
NodeIDFn func() uint64
RetentionPolicyFn func(database, name string) (*meta.RetentionPolicyInfo, error)
CreateShardGroupIfNotExistsFn func(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
DatabaseFn func(database string) *meta.DatabaseInfo
ShardOwnerFn func(shardID uint64) (string, string, *meta.ShardGroupInfo)
}
func (m PointsWriterMetaClient) NodeID() uint64 { return m.NodeIDFn() }
func (m PointsWriterMetaClient) RetentionPolicy(database, name string) (*meta.RetentionPolicyInfo, error) {
return m.RetentionPolicyFn(database, name)
}
func (m PointsWriterMetaClient) CreateShardGroup(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) {
return m.CreateShardGroupIfNotExistsFn(database, policy, timestamp)
}
func (m PointsWriterMetaClient) Database(database string) *meta.DatabaseInfo {
return m.DatabaseFn(database)
}
func (m PointsWriterMetaClient) ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo) {
return m.ShardOwnerFn(shardID)
}
type Subscriber struct {
PointsFn func() chan<- *coordinator.WritePointsRequest
}
func (s Subscriber) Points() chan<- *coordinator.WritePointsRequest {
return s.PointsFn()
}
func NewRetentionPolicy(name string, duration time.Duration, nodeCount int) *meta.RetentionPolicyInfo {
shards := []meta.ShardInfo{}
owners := []meta.ShardOwner{}
for i := 1; i <= nodeCount; i++ {
owners = append(owners, meta.ShardOwner{NodeID: uint64(i)})
}
// each node is fully replicated with each other
shards = append(shards, meta.ShardInfo{
ID: nextShardID(),
Owners: owners,
})
start := time.Now()
rp := &meta.RetentionPolicyInfo{
Name: "myrp",
ReplicaN: nodeCount,
Duration: duration,
ShardGroupDuration: duration,
ShardGroups: []meta.ShardGroupInfo{
meta.ShardGroupInfo{
ID: nextShardID(),
StartTime: start,
EndTime: start.Add(duration).Add(-1),
Shards: shards,
},
},
}
return rp
}
func AttachShardGroupInfo(rp *meta.RetentionPolicyInfo, owners []meta.ShardOwner) {
var startTime, endTime time.Time
if len(rp.ShardGroups) == 0 {
startTime = time.Now()
} else {
startTime = rp.ShardGroups[len(rp.ShardGroups)-1].StartTime.Add(rp.ShardGroupDuration)
}
endTime = startTime.Add(rp.ShardGroupDuration).Add(-1)
sh := meta.ShardGroupInfo{
ID: uint64(len(rp.ShardGroups) + 1),
StartTime: startTime,
EndTime: endTime,
Shards: []meta.ShardInfo{
meta.ShardInfo{
ID: nextShardID(),
Owners: owners,
},
},
}
rp.ShardGroups = append(rp.ShardGroups, sh)
}
func nextShardID() uint64 {
return atomic.AddUint64(&shardID, 1)
}

255
v1/coordinator/shard_mapper.go Normal file
View File

@ -0,0 +1,255 @@
package coordinator
import (
"context"
"io"
"time"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxdb/v2/v1/tsdb"
"github.com/influxdata/influxql"
)
// IteratorCreator is an interface that combines mapping fields and creating iterators.
type IteratorCreator interface {
query.IteratorCreator
influxql.FieldMapper
io.Closer
}
// LocalShardMapper implements a ShardMapper for local shards.
type LocalShardMapper struct {
MetaClient interface {
ShardGroupsByTimeRange(database, policy string, min, max time.Time) (a []meta.ShardGroupInfo, err error)
}
TSDBStore interface {
ShardGroup(ids []uint64) tsdb.ShardGroup
}
}
// MapShards maps the sources to the appropriate shards into an IteratorCreator.
func (e *LocalShardMapper) MapShards(sources influxql.Sources, t influxql.TimeRange, opt query.SelectOptions) (query.ShardGroup, error) {
a := &LocalShardMapping{
ShardMap: make(map[Source]tsdb.ShardGroup),
}
tmin := time.Unix(0, t.MinTimeNano())
tmax := time.Unix(0, t.MaxTimeNano())
if err := e.mapShards(a, sources, tmin, tmax); err != nil {
return nil, err
}
a.MinTime, a.MaxTime = tmin, tmax
return a, nil
}
func (e *LocalShardMapper) mapShards(a *LocalShardMapping, sources influxql.Sources, tmin, tmax time.Time) error {
for _, s := range sources {
switch s := s.(type) {
case *influxql.Measurement:
source := Source{
Database: s.Database,
RetentionPolicy: s.RetentionPolicy,
}
// Retrieve the list of shards for this database. This list of
// shards is always the same regardless of which measurement we are
// using.
if _, ok := a.ShardMap[source]; !ok {
groups, err := e.MetaClient.ShardGroupsByTimeRange(s.Database, s.RetentionPolicy, tmin, tmax)
if err != nil {
return err
}
if len(groups) == 0 {
a.ShardMap[source] = nil
continue
}
shardIDs := make([]uint64, 0, len(groups[0].Shards)*len(groups))
for _, g := range groups {
for _, si := range g.Shards {
shardIDs = append(shardIDs, si.ID)
}
}
a.ShardMap[source] = e.TSDBStore.ShardGroup(shardIDs)
}
case *influxql.SubQuery:
if err := e.mapShards(a, s.Statement.Sources, tmin, tmax); err != nil {
return err
}
}
}
return nil
}
// ShardMapper maps data sources to a list of shard information.
type LocalShardMapping struct {
ShardMap map[Source]tsdb.ShardGroup
// MinTime is the minimum time that this shard mapper will allow.
// Any attempt to use a time before this one will automatically result in using
// this time instead.
MinTime time.Time
// MaxTime is the maximum time that this shard mapper will allow.
// Any attempt to use a time after this one will automatically result in using
// this time instead.
MaxTime time.Time
}
func (a *LocalShardMapping) FieldDimensions(m *influxql.Measurement) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) {
source := Source{
Database: m.Database,
RetentionPolicy: m.RetentionPolicy,
}
sg := a.ShardMap[source]
if sg == nil {
return
}
fields = make(map[string]influxql.DataType)
dimensions = make(map[string]struct{})
var measurements []string
if m.Regex != nil {
measurements = sg.MeasurementsByRegex(m.Regex.Val)
} else {
measurements = []string{m.Name}
}
f, d, err := sg.FieldDimensions(measurements)
if err != nil {
return nil, nil, err
}
for k, typ := range f {
fields[k] = typ
}
for k := range d {
dimensions[k] = struct{}{}
}
return
}
func (a *LocalShardMapping) MapType(m *influxql.Measurement, field string) influxql.DataType {
source := Source{
Database: m.Database,
RetentionPolicy: m.RetentionPolicy,
}
sg := a.ShardMap[source]
if sg == nil {
return influxql.Unknown
}
var names []string
if m.Regex != nil {
names = sg.MeasurementsByRegex(m.Regex.Val)
} else {
names = []string{m.Name}
}
var typ influxql.DataType
for _, name := range names {
if m.SystemIterator != "" {
name = m.SystemIterator
}
t := sg.MapType(name, field)
if typ.LessThan(t) {
typ = t
}
}
return typ
}
func (a *LocalShardMapping) CreateIterator(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) {
source := Source{
Database: m.Database,
RetentionPolicy: m.RetentionPolicy,
}
sg := a.ShardMap[source]
if sg == nil {
return nil, nil
}
// Override the time constraints if they don't match each other.
if !a.MinTime.IsZero() && opt.StartTime < a.MinTime.UnixNano() {
opt.StartTime = a.MinTime.UnixNano()
}
if !a.MaxTime.IsZero() && opt.EndTime > a.MaxTime.UnixNano() {
opt.EndTime = a.MaxTime.UnixNano()
}
if m.Regex != nil {
measurements := sg.MeasurementsByRegex(m.Regex.Val)
inputs := make([]query.Iterator, 0, len(measurements))
if err := func() error {
// Create a Measurement for each returned matching measurement value
// from the regex.
for _, measurement := range measurements {
mm := m.Clone()
mm.Name = measurement // Set the name to this matching regex value.
input, err := sg.CreateIterator(ctx, mm, opt)
if err != nil {
return err
}
inputs = append(inputs, input)
}
return nil
}(); err != nil {
query.Iterators(inputs).Close()
return nil, err
}
return query.Iterators(inputs).Merge(opt)
}
return sg.CreateIterator(ctx, m, opt)
}
func (a *LocalShardMapping) IteratorCost(m *influxql.Measurement, opt query.IteratorOptions) (query.IteratorCost, error) {
source := Source{
Database: m.Database,
RetentionPolicy: m.RetentionPolicy,
}
sg := a.ShardMap[source]
if sg == nil {
return query.IteratorCost{}, nil
}
// Override the time constraints if they don't match each other.
if !a.MinTime.IsZero() && opt.StartTime < a.MinTime.UnixNano() {
opt.StartTime = a.MinTime.UnixNano()
}
if !a.MaxTime.IsZero() && opt.EndTime > a.MaxTime.UnixNano() {
opt.EndTime = a.MaxTime.UnixNano()
}
if m.Regex != nil {
var costs query.IteratorCost
measurements := sg.MeasurementsByRegex(m.Regex.Val)
for _, measurement := range measurements {
cost, err := sg.IteratorCost(measurement, opt)
if err != nil {
return query.IteratorCost{}, err
}
costs = costs.Combine(cost)
}
return costs, nil
}
return sg.IteratorCost(m.Name, opt)
}
// Close clears out the list of mapped shards.
func (a *LocalShardMapping) Close() error {
a.ShardMap = nil
return nil
}
// Source contains the database and retention policy source for data.
type Source struct {
Database string
RetentionPolicy string
}

105
v1/coordinator/shard_mapper_test.go Normal file
View File

@ -0,0 +1,105 @@
package coordinator_test
import (
"context"
"reflect"
"testing"
"time"
"github.com/influxdata/influxdb/v2/influxql/query"
"github.com/influxdata/influxdb/v2/v1/coordinator"
"github.com/influxdata/influxdb/v2/v1/internal"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxdb/v2/v1/tsdb"
"github.com/influxdata/influxql"
)
func TestLocalShardMapper(t *testing.T) {
var metaClient MetaClient
metaClient.ShardGroupsByTimeRangeFn = func(database, policy string, min, max time.Time) ([]meta.ShardGroupInfo, error) {
if database != "db0" {
t.Errorf("unexpected database: %s", database)
}
if policy != "rp0" {
t.Errorf("unexpected retention policy: %s", policy)
}
return []meta.ShardGroupInfo{
{ID: 1, Shards: []meta.ShardInfo{
{ID: 1, Owners: []meta.ShardOwner{{NodeID: 0}}},
{ID: 2, Owners: []meta.ShardOwner{{NodeID: 0}}},
}},
{ID: 2, Shards: []meta.ShardInfo{
{ID: 3, Owners: []meta.ShardOwner{{NodeID: 0}}},
{ID: 4, Owners: []meta.ShardOwner{{NodeID: 0}}},
}},
}, nil
}
tsdbStore := &internal.TSDBStoreMock{}
tsdbStore.ShardGroupFn = func(ids []uint64) tsdb.ShardGroup {
if !reflect.DeepEqual(ids, []uint64{1, 2, 3, 4}) {
t.Errorf("unexpected shard ids: %#v", ids)
}
var sh MockShard
sh.CreateIteratorFn = func(ctx context.Context, measurement *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) {
if measurement.Name != "cpu" {
t.Errorf("unexpected measurement: %s", measurement.Name)
}
return &FloatIterator{}, nil
}
return &sh
}
// Initialize the shard mapper.
shardMapper := &coordinator.LocalShardMapper{
MetaClient: &metaClient,
TSDBStore: tsdbStore,
}
// Normal measurement.
measurement := &influxql.Measurement{
Database: "db0",
RetentionPolicy: "rp0",
Name: "cpu",
}
ic, err := shardMapper.MapShards([]influxql.Source{measurement}, influxql.TimeRange{}, query.SelectOptions{})
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
// This should be a LocalShardMapping.
m, ok := ic.(*coordinator.LocalShardMapping)
if !ok {
t.Fatalf("unexpected mapping type: %T", ic)
} else if len(m.ShardMap) != 1 {
t.Fatalf("unexpected number of shard mappings: %d", len(m.ShardMap))
}
if _, err := ic.CreateIterator(context.Background(), measurement, query.IteratorOptions{}); err != nil {
t.Fatalf("unexpected error: %s", err)
}
// Subquery.
subquery := &influxql.SubQuery{
Statement: &influxql.SelectStatement{
Sources: []influxql.Source{measurement},
},
}
ic, err = shardMapper.MapShards([]influxql.Source{subquery}, influxql.TimeRange{}, query.SelectOptions{})
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
// This should be a LocalShardMapping.
m, ok = ic.(*coordinator.LocalShardMapping)
if !ok {
t.Fatalf("unexpected mapping type: %T", ic)
} else if len(m.ShardMap) != 1 {
t.Fatalf("unexpected number of shard mappings: %d", len(m.ShardMap))
}
if _, err := ic.CreateIterator(context.Background(), measurement, query.IteratorOptions{}); err != nil {
t.Fatalf("unexpected error: %s", err)
}
}

Some files were not shown because too many files have changed in this diff Show More