From ea6262b7120759f9db52d52223d3896c5d49d337 Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Wed, 6 Apr 2016 16:52:39 -0400 Subject: [PATCH] Enhance comparing tags and fields in the where clause Now it is possible to compare tags and fields and it is also now possible to compare tags and tags. Previously, it was only possible to compare fields with fields and tags with a string or a regex. Fixes #3371. --- cmd/influxd/run/server_test.go | 12 ++++++++++++ tsdb/engine/tsm1/engine.go | 24 +++++++++++++++--------- tsdb/engine/tsm1/iterator.gen.go | 16 ++++++++-------- tsdb/engine/tsm1/iterator.gen.go.tmpl | 4 ++-- tsdb/meta.go | 25 +++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 19 deletions(-) diff --git a/cmd/influxd/run/server_test.go b/cmd/influxd/run/server_test.go index e743595c50..3c3ef886f6 100644 --- a/cmd/influxd/run/server_test.go +++ b/cmd/influxd/run/server_test.go @@ -4577,6 +4577,18 @@ func TestServer_Query_Where_With_Tags(t *testing.T) { command: `show series where data-center = 'foo'`, exp: `{"error":"error parsing query: found DATA, expected identifier, string, number, bool at line 1, char 19"}`, }, + &Query{ + name: "where comparing tag and field", + params: url.Values{"db": []string{"db0"}}, + command: `select foo from where_events where tennant != foo`, + exp: `{"results":[{"series":[{"name":"where_events","columns":["time","foo"],"values":[["2009-11-10T23:00:02Z","bar"],["2009-11-10T23:00:03Z","baz"],["2009-11-10T23:00:04Z","bat"],["2009-11-10T23:00:05Z","bar"],["2009-11-10T23:00:06Z","bap"]]}]}]}`, + }, + &Query{ + name: "where comparing tag and tag", + params: url.Values{"db": []string{"db0"}}, + command: `select foo from where_events where tennant = tennant`, + exp: `{"results":[{"series":[{"name":"where_events","columns":["time","foo"],"values":[["2009-11-10T23:00:02Z","bar"],["2009-11-10T23:00:03Z","baz"],["2009-11-10T23:00:04Z","bat"],["2009-11-10T23:00:05Z","bar"],["2009-11-10T23:00:06Z","bap"]]}]}]}`, + }, }...) for i, query := range test.queries { diff --git a/tsdb/engine/tsm1/engine.go b/tsdb/engine/tsm1/engine.go index f6714439b3..543101550c 100644 --- a/tsdb/engine/tsm1/engine.go +++ b/tsdb/engine/tsm1/engine.go @@ -806,10 +806,8 @@ func (e *Engine) createVarRefIterator(opt influxql.IteratorOptions) ([]influxql. if t.Filters[i] != nil { // Retrieve non-time fields from this series filter and filter out tags. for _, f := range influxql.ExprNames(t.Filters[i]) { - if mm.HasField(f) { - conditionFields[fields] = f - fields++ - } + conditionFields[fields] = f + fields++ } } @@ -865,15 +863,23 @@ func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, mm *tsdb.Measu // Build conditional field cursors. // If a conditional field doesn't exist then ignore the series. - var conds []*bufCursor + var conds []cursorAt if len(conditionFields) > 0 { - conds = make([]*bufCursor, len(conditionFields)) + conds = make([]cursorAt, len(conditionFields)) for i := range conds { cur := e.buildCursor(mm.Name, seriesKey, conditionFields[i], opt) - if cur == nil { - return nil, nil + if cur != nil { + conds[i] = newBufCursor(cur, opt.Ascending) + continue + } + + // If field doesn't exist, use the tag value. + // However, if the tag value is blank then return a null. + if v := tags.Value(conditionFields[i]); v == "" { + conds[i] = &stringNilLiteralCursor{} + } else { + conds[i] = &stringLiteralCursor{value: v} } - conds[i] = newBufCursor(cur, opt.Ascending) } } diff --git a/tsdb/engine/tsm1/iterator.gen.go b/tsdb/engine/tsm1/iterator.gen.go index 9d2c8731b1..6022bd40b7 100644 --- a/tsdb/engine/tsm1/iterator.gen.go +++ b/tsdb/engine/tsm1/iterator.gen.go @@ -112,7 +112,7 @@ type floatIterator struct { aux []cursorAt conds struct { names []string - curs []*bufCursor + curs []cursorAt } opt influxql.IteratorOptions @@ -124,7 +124,7 @@ type floatIterator struct { statsBuf influxql.IteratorStats } -func newFloatIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur floatCursor, aux []cursorAt, conds []*bufCursor, condNames []string) *floatIterator { +func newFloatIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur floatCursor, aux []cursorAt, conds []cursorAt, condNames []string) *floatIterator { itr := &floatIterator{ cur: cur, aux: aux, @@ -486,7 +486,7 @@ type integerIterator struct { aux []cursorAt conds struct { names []string - curs []*bufCursor + curs []cursorAt } opt influxql.IteratorOptions @@ -498,7 +498,7 @@ type integerIterator struct { statsBuf influxql.IteratorStats } -func newIntegerIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur integerCursor, aux []cursorAt, conds []*bufCursor, condNames []string) *integerIterator { +func newIntegerIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur integerCursor, aux []cursorAt, conds []cursorAt, condNames []string) *integerIterator { itr := &integerIterator{ cur: cur, aux: aux, @@ -860,7 +860,7 @@ type stringIterator struct { aux []cursorAt conds struct { names []string - curs []*bufCursor + curs []cursorAt } opt influxql.IteratorOptions @@ -872,7 +872,7 @@ type stringIterator struct { statsBuf influxql.IteratorStats } -func newStringIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur stringCursor, aux []cursorAt, conds []*bufCursor, condNames []string) *stringIterator { +func newStringIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur stringCursor, aux []cursorAt, conds []cursorAt, condNames []string) *stringIterator { itr := &stringIterator{ cur: cur, aux: aux, @@ -1234,7 +1234,7 @@ type booleanIterator struct { aux []cursorAt conds struct { names []string - curs []*bufCursor + curs []cursorAt } opt influxql.IteratorOptions @@ -1246,7 +1246,7 @@ type booleanIterator struct { statsBuf influxql.IteratorStats } -func newBooleanIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur booleanCursor, aux []cursorAt, conds []*bufCursor, condNames []string) *booleanIterator { +func newBooleanIterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur booleanCursor, aux []cursorAt, conds []cursorAt, condNames []string) *booleanIterator { itr := &booleanIterator{ cur: cur, aux: aux, diff --git a/tsdb/engine/tsm1/iterator.gen.go.tmpl b/tsdb/engine/tsm1/iterator.gen.go.tmpl index d50d3500b9..0f0770a19d 100644 --- a/tsdb/engine/tsm1/iterator.gen.go.tmpl +++ b/tsdb/engine/tsm1/iterator.gen.go.tmpl @@ -108,7 +108,7 @@ type {{.name}}Iterator struct { aux []cursorAt conds struct { names []string - curs []*bufCursor + curs []cursorAt } opt influxql.IteratorOptions @@ -120,7 +120,7 @@ type {{.name}}Iterator struct { statsBuf influxql.IteratorStats } -func new{{.Name}}Iterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur {{.name}}Cursor, aux []cursorAt, conds []*bufCursor, condNames []string) *{{.name}}Iterator { +func new{{.Name}}Iterator(name string, tags influxql.Tags, opt influxql.IteratorOptions, cur {{.name}}Cursor, aux []cursorAt, conds []cursorAt, condNames []string) *{{.name}}Iterator { itr := &{{.name}}Iterator{ cur: cur, aux: aux, diff --git a/tsdb/meta.go b/tsdb/meta.go index eb88d94c99..efc61c7fa8 100644 --- a/tsdb/meta.go +++ b/tsdb/meta.go @@ -761,6 +761,11 @@ func (m *Measurement) idsForExpr(n *influxql.BinaryExpr) (SeriesIDs, influxql.Ex // the expression passed in, as the filter. if name.Val != "_name" && m.hasField(name.Val) { return m.seriesIDs, n, nil + } else if value, ok := value.(*influxql.VarRef); ok { + // Check if the RHS is a variable and if it is a field. + if value.Val != "_name" && m.hasField(value.Val) { + return m.seriesIDs, n, nil + } } // Retrieve list of series with this tag key. @@ -850,6 +855,26 @@ func (m *Measurement) idsForExpr(n *influxql.BinaryExpr) (SeriesIDs, influxql.Ex return ids, &influxql.BooleanLiteral{Val: true}, nil } + // compare tag values + if ref, ok := value.(*influxql.VarRef); ok { + var ids SeriesIDs + + if n.Op == influxql.NEQ { + ids = m.seriesIDs + } + + rhsTagVals := m.seriesByTagKeyValue[ref.Val] + for k := range tagVals { + tags := tagVals[k].Intersect(rhsTagVals[k]) + if n.Op == influxql.EQ { + ids = ids.Union(tags) + } else if n.Op == influxql.NEQ { + ids = ids.Reject(tags) + } + } + return ids, &influxql.BooleanLiteral{Val: true}, nil + } + if n.Op == influxql.NEQ || n.Op == influxql.NEQREGEX { return m.seriesIDs, &influxql.BooleanLiteral{Val: true}, nil }