From 2e7816ebd9d0694c5287dc26f1bff706c14706bf Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Wed, 16 Mar 2016 20:55:50 -0400 Subject: [PATCH] Support the distinct() call for booleans Normalize the time for the distinct() call to either be at the beginning of the group by interval or the start time similar to every other call. The timestamp previously just showed the first time found and didn't make a lot of sense in the context of what the function was supposed to do. Fixes #6040. --- cmd/influxd/run/server_test.go | 10 ++++---- influxql/call_iterator.go | 23 ++++++++++++++++++ influxql/select.go | 6 ++++- influxql/select_test.go | 44 +++++++++++++++++++++------------- 4 files changed, 61 insertions(+), 22 deletions(-) diff --git a/cmd/influxd/run/server_test.go b/cmd/influxd/run/server_test.go index 16faaea4e4..c041d5ce26 100644 --- a/cmd/influxd/run/server_test.go +++ b/cmd/influxd/run/server_test.go @@ -2285,13 +2285,13 @@ func TestServer_Query_Aggregates_IntMany(t *testing.T) { name: "distinct as call - int", params: url.Values{"db": []string{"db0"}}, command: `SELECT DISTINCT(value) FROM intmany`, - exp: `{"results":[{"series":[{"name":"intmany","columns":["time","distinct"],"values":[["2000-01-01T00:00:00Z",2],["2000-01-01T00:00:10Z",4],["2000-01-01T00:00:40Z",5],["2000-01-01T00:01:00Z",7],["2000-01-01T00:01:10Z",9]]}]}]}`, + exp: `{"results":[{"series":[{"name":"intmany","columns":["time","distinct"],"values":[["1970-01-01T00:00:00Z",2],["1970-01-01T00:00:00Z",4],["1970-01-01T00:00:00Z",5],["1970-01-01T00:00:00Z",7],["1970-01-01T00:00:00Z",9]]}]}]}`, }, &Query{ name: "distinct alt syntax - int", params: url.Values{"db": []string{"db0"}}, command: `SELECT DISTINCT value FROM intmany`, - exp: `{"results":[{"series":[{"name":"intmany","columns":["time","distinct"],"values":[["2000-01-01T00:00:00Z",2],["2000-01-01T00:00:10Z",4],["2000-01-01T00:00:40Z",5],["2000-01-01T00:01:00Z",7],["2000-01-01T00:01:10Z",9]]}]}]}`, + exp: `{"results":[{"series":[{"name":"intmany","columns":["time","distinct"],"values":[["1970-01-01T00:00:00Z",2],["1970-01-01T00:00:00Z",4],["1970-01-01T00:00:00Z",5],["1970-01-01T00:00:00Z",7],["1970-01-01T00:00:00Z",9]]}]}]}`, }, &Query{ name: "distinct select tag - int", @@ -2659,13 +2659,13 @@ func TestServer_Query_Aggregates_FloatMany(t *testing.T) { name: "distinct as call - float", params: url.Values{"db": []string{"db0"}}, command: `SELECT DISTINCT(value) FROM floatmany`, - exp: `{"results":[{"series":[{"name":"floatmany","columns":["time","distinct"],"values":[["2000-01-01T00:00:00Z",2],["2000-01-01T00:00:10Z",4],["2000-01-01T00:00:40Z",5],["2000-01-01T00:01:00Z",7],["2000-01-01T00:01:10Z",9]]}]}]}`, + exp: `{"results":[{"series":[{"name":"floatmany","columns":["time","distinct"],"values":[["1970-01-01T00:00:00Z",2],["1970-01-01T00:00:00Z",4],["1970-01-01T00:00:00Z",5],["1970-01-01T00:00:00Z",7],["1970-01-01T00:00:00Z",9]]}]}]}`, }, &Query{ name: "distinct alt syntax - float", params: url.Values{"db": []string{"db0"}}, command: `SELECT DISTINCT value FROM floatmany`, - exp: `{"results":[{"series":[{"name":"floatmany","columns":["time","distinct"],"values":[["2000-01-01T00:00:00Z",2],["2000-01-01T00:00:10Z",4],["2000-01-01T00:00:40Z",5],["2000-01-01T00:01:00Z",7],["2000-01-01T00:01:10Z",9]]}]}]}`, + exp: `{"results":[{"series":[{"name":"floatmany","columns":["time","distinct"],"values":[["1970-01-01T00:00:00Z",2],["1970-01-01T00:00:00Z",4],["1970-01-01T00:00:00Z",5],["1970-01-01T00:00:00Z",7],["1970-01-01T00:00:00Z",9]]}]}]}`, }, &Query{ name: "distinct select tag - float", @@ -3134,7 +3134,7 @@ func TestServer_Query_AggregateSelectors(t *testing.T) { name: "distinct - baseline 30s", params: url.Values{"db": []string{"db0"}}, command: `SELECT distinct(rx) FROM network where time >= '2000-01-01T00:00:00Z' AND time <= '2000-01-01T00:01:29Z' group by time(30s)`, - exp: `{"results":[{"series":[{"name":"network","columns":["time","distinct"],"values":[["2000-01-01T00:00:00Z",10],["2000-01-01T00:00:10Z",40],["2000-01-01T00:00:30Z",40],["2000-01-01T00:00:40Z",50],["2000-01-01T00:01:00Z",70],["2000-01-01T00:01:10Z",90],["2000-01-01T00:01:20Z",5]]}]}]}`, + exp: `{"results":[{"series":[{"name":"network","columns":["time","distinct"],"values":[["2000-01-01T00:00:00Z",10],["2000-01-01T00:00:00Z",40],["2000-01-01T00:00:30Z",40],["2000-01-01T00:00:30Z",50],["2000-01-01T00:01:00Z",70],["2000-01-01T00:01:00Z",90],["2000-01-01T00:01:00Z",5]]}]}]}`, }, &Query{ name: "distinct - time", diff --git a/influxql/call_iterator.go b/influxql/call_iterator.go index 19f9d17206..452c156da6 100644 --- a/influxql/call_iterator.go +++ b/influxql/call_iterator.go @@ -379,6 +379,12 @@ func NewDistinctIterator(input Iterator, opt IteratorOptions) (Iterator, error) return fn, fn } return &stringReduceStringIterator{input: newBufStringIterator(input), opt: opt, create: createFn}, nil + case BooleanIterator: + createFn := func() (BooleanPointAggregator, BooleanPointEmitter) { + fn := NewBooleanSliceFuncReducer(BooleanDistinctReduceSlice) + return fn, fn + } + return &booleanReduceBooleanIterator{input: newBufBooleanIterator(input), opt: opt, create: createFn}, nil default: return nil, fmt.Errorf("unsupported distinct iterator type: %T", input) } @@ -435,6 +441,23 @@ func StringDistinctReduceSlice(a []StringPoint) []StringPoint { return points } +// BooleanDistinctReduceSlice returns the distinct value within a window. +func BooleanDistinctReduceSlice(a []BooleanPoint) []BooleanPoint { + m := make(map[bool]BooleanPoint) + for _, p := range a { + if _, ok := m[p.Value]; !ok { + m[p.Value] = p + } + } + + points := make([]BooleanPoint, 0, len(m)) + for _, p := range m { + points = append(points, BooleanPoint{Time: p.Time, Value: p.Value}) + } + sort.Sort(booleanPoints(points)) + return points +} + // newMeanIterator returns an iterator for operating on a mean() call. func newMeanIterator(input Iterator, opt IteratorOptions) (Iterator, error) { switch input := input.(type) { diff --git a/influxql/select.go b/influxql/select.go index 78ff48a4f4..870b2dd18d 100644 --- a/influxql/select.go +++ b/influxql/select.go @@ -214,7 +214,11 @@ func buildExprIterator(expr Expr, ic IteratorCreator, opt IteratorOptions) (Iter if err != nil { return nil, err } - return NewDistinctIterator(input, opt) + input, err = NewDistinctIterator(input, opt) + if err != nil { + return nil, err + } + return NewIntervalIterator(input, opt), nil case "derivative", "non_negative_derivative": input, err := buildExprIterator(expr.Args[0], ic, opt) if err != nil { diff --git a/influxql/select_test.go b/influxql/select_test.go index 62a1214338..1fb0c2056c 100644 --- a/influxql/select_test.go +++ b/influxql/select_test.go @@ -66,8 +66,8 @@ func TestSelect_Distinct_Float(t *testing.T) { t.Fatal(err) } else if a := Iterators(itrs).ReadAll(); !deep.Equal(a, [][]influxql.Point{ {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 20}}, - {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1 * Second, Value: 19}}, - {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 5 * Second, Value: 10}}, + {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 19}}, + {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 0 * Second, Value: 10}}, {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: 2}}, }) { t.Fatalf("unexpected points: %s", spew.Sdump(a)) @@ -95,8 +95,8 @@ func TestSelect_Distinct_Integer(t *testing.T) { t.Fatal(err) } else if a := Iterators(itrs).ReadAll(); !deep.Equal(a, [][]influxql.Point{ {&influxql.IntegerPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 20}}, - {&influxql.IntegerPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1 * Second, Value: 19}}, - {&influxql.IntegerPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 5 * Second, Value: 10}}, + {&influxql.IntegerPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 19}}, + {&influxql.IntegerPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 0 * Second, Value: 10}}, {&influxql.IntegerPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: 2}}, }) { t.Fatalf("unexpected points: %s", spew.Sdump(a)) @@ -124,29 +124,41 @@ func TestSelect_Distinct_String(t *testing.T) { t.Fatal(err) } else if a := Iterators(itrs).ReadAll(); !deep.Equal(a, [][]influxql.Point{ {&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: "a"}}, - {&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1 * Second, Value: "b"}}, - {&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 5 * Second, Value: "c"}}, + {&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: "b"}}, + {&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 0 * Second, Value: "c"}}, {&influxql.StringPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: "d"}}, }) { t.Fatalf("unexpected points: %s", spew.Sdump(a)) } } -// Ensure a SELECT distinct() query cannot be executed on booleans. +// Ensure a SELECT distinct() query can be executed. func TestSelect_Distinct_Boolean(t *testing.T) { var ic IteratorCreator ic.CreateIteratorFn = func(opt influxql.IteratorOptions) (influxql.Iterator, error) { - return &BooleanIterator{}, nil + return &BooleanIterator{Points: []influxql.BooleanPoint{ + {Name: "cpu", Tags: ParseTags("region=west,host=A"), Time: 0 * Second, Value: true}, + {Name: "cpu", Tags: ParseTags("region=west,host=A"), Time: 1 * Second, Value: false}, + {Name: "cpu", Tags: ParseTags("region=west,host=B"), Time: 5 * Second, Value: false}, + {Name: "cpu", Tags: ParseTags("region=east,host=A"), Time: 9 * Second, Value: true}, + {Name: "cpu", Tags: ParseTags("region=east,host=A"), Time: 10 * Second, Value: false}, + {Name: "cpu", Tags: ParseTags("region=east,host=A"), Time: 11 * Second, Value: false}, + {Name: "cpu", Tags: ParseTags("region=east,host=A"), Time: 12 * Second, Value: true}, + }}, nil } // Execute selection. itrs, err := influxql.Select(MustParseSelectStatement(`SELECT distinct(value) FROM cpu WHERE time >= '1970-01-01T00:00:00Z' AND time < '1970-01-02T00:00:00Z' GROUP BY time(10s), host fill(none)`), &ic, nil) - if err == nil || err.Error() != "unsupported distinct iterator type: *influxql_test.BooleanIterator" { - t.Errorf("unexpected error: %s", err) - } - - if itrs != nil { - influxql.Iterators(itrs).Close() + if err != nil { + t.Fatal(err) + } else if a := Iterators(itrs).ReadAll(); !deep.Equal(a, [][]influxql.Point{ + {&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: true}}, + {&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: false}}, + {&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 0 * Second, Value: false}}, + {&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: false}}, + {&influxql.BooleanPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: true}}, + }) { + t.Errorf("unexpected points: %s", spew.Sdump(a)) } } @@ -1683,8 +1695,8 @@ func TestSelect_ParenExpr(t *testing.T) { t.Fatal(err) } else if a := Iterators(itrs).ReadAll(); !deep.Equal(a, [][]influxql.Point{ {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 20}}, - {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 1 * Second, Value: 19}}, - {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 5 * Second, Value: 10}}, + {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 0 * Second, Value: 19}}, + {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=B"), Time: 0 * Second, Value: 10}}, {&influxql.FloatPoint{Name: "cpu", Tags: ParseTags("host=A"), Time: 10 * Second, Value: 2}}, }) { t.Fatalf("unexpected points: %s", spew.Sdump(a))