Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #7442 from influxdata/js-5955-make-regex-work-on-field-keys-in-select 

Support using regexes to select fields and dimensions
pull/7470/head
Jonathan A. Sternberg 2016-10-17 11:37:47 -05:00 committed by GitHub
parent 5c30c23d1b b60b4b371e
commit 3496c5b85f
4 changed files with 112 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -23,6 +23,7 @@
- [#7388](https://github.com/influxdata/influxdb/pull/7388): Implement cumulative_sum() function.
- [#7441](https://github.com/influxdata/influxdb/pull/7441): Speed up shutdown by closing shards concurrently.
- [#7146](https://github.com/influxdata/influxdb/issues/7146): Add max-values-per-tag to limit high tag cardinality data
- [#5955](https://github.com/influxdata/influxdb/issues/5955): Make regex work on field and dimension keys in SELECT clause.
### Bugfixes

47
influxql/ast.go
View File

@ -1128,6 +1128,12 @@ func (s *SelectStatement) RewriteFields(ic IteratorCreator) (*SelectStatement, e
}
rwFields = append(rwFields, &Field{Expr: &VarRef{Val: ref.Val, Type: ref.Type}})
}
case *RegexLiteral:
for _, ref := range fields {
if expr.Val.MatchString(ref.Val) {
rwFields = append(rwFields, &Field{Expr: &VarRef{Val: ref.Val, Type: ref.Type}})
}
}
case *Call:
// Clone a template that we can modify and use for new fields.
template := CloneExpr(expr).(*Call)
@ -1149,10 +1155,16 @@ func (s *SelectStatement) RewriteFields(ic IteratorCreator) (*SelectStatement, e
continue
}
wc, ok := call.Args[0].(*Wildcard)
if ok && wc.Type == TAG {
return s, fmt.Errorf("unable to use tag wildcard in %s()", call.Name)
} else if !ok {
// Retrieve if this is a wildcard or a regular expression.
var re *regexp.Regexp
switch expr := call.Args[0].(type) {
case *Wildcard:
if expr.Type == TAG {
return s, fmt.Errorf("unable to use tag wildcard in %s()", call.Name)
}
case *RegexLiteral:
re = expr.Val
default:
rwFields = append(rwFields, f)
continue
}
@ -1179,6 +1191,8 @@ func (s *SelectStatement) RewriteFields(ic IteratorCreator) (*SelectStatement, e
continue
} else if _, ok := supportedTypes[ref.Type]; !ok {
continue
} else if re != nil && !re.MatchString(ref.Val) {
continue
}
// Make a new expression and replace the wildcard within this cloned expression.
@ -1200,11 +1214,17 @@ func (s *SelectStatement) RewriteFields(ic IteratorCreator) (*SelectStatement, e
// Allocate a slice assuming there is exactly one wildcard for efficiency.
rwDimensions := make(Dimensions, 0, len(s.Dimensions)+len(dimensions)-1)
for _, d := range s.Dimensions {
switch d.Expr.(type) {
switch expr := d.Expr.(type) {
case *Wildcard:
for _, name := range dimensions {
rwDimensions = append(rwDimensions, &Dimension{Expr: &VarRef{Val: name}})
}
case *RegexLiteral:
for _, name := range dimensions {
if expr.Val.MatchString(name) {
rwDimensions = append(rwDimensions, &Dimension{Expr: &VarRef{Val: name}})
}
}
default:
rwDimensions = append(rwDimensions, d)
}
@ -1418,8 +1438,8 @@ func (s *SelectStatement) HasFieldWildcard() (hasWildcard bool) {
if hasWildcard {
return
}
_, ok := n.(*Wildcard)
if ok {
switch n.(type) {
case *Wildcard, *RegexLiteral:
hasWildcard = true
}
})
@ -1430,8 +1450,8 @@ func (s *SelectStatement) HasFieldWildcard() (hasWildcard bool) {
// at least 1 wildcard in the dimensions aka `GROUP BY`
func (s *SelectStatement) HasDimensionWildcard() bool {
for _, d := range s.Dimensions {
_, ok := d.Expr.(*Wildcard)
if ok {
switch d.Expr.(type) {
case *Wildcard, *RegexLiteral:
return true
}
}
@ -1513,6 +1533,7 @@ func (s *SelectStatement) validateDimensions() error {
return errors.New("time() is a function and expects at least one argument")
}
case *Wildcard:
case *RegexLiteral:
default:
return errors.New("only time and tag dimensions allowed")
}
@ -1599,7 +1620,7 @@ func (s *SelectStatement) validPercentileAggr(expr *Call) error {
}
switch expr.Args[0].(type) {
case *VarRef:
case *VarRef, *RegexLiteral, *Wildcard:
// do nothing
default:
return fmt.Errorf("expected field argument in percentile()")
@ -1623,7 +1644,7 @@ func (s *SelectStatement) validSampleAggr(expr *Call) error {
}
switch expr.Args[0].(type) {
case *VarRef:
case *VarRef, *RegexLiteral, *Wildcard:
// do nothing
default:
return fmt.Errorf("expected field argument in sample()")
@ -1704,7 +1725,7 @@ func (s *SelectStatement) validateAggregates(tr targetRequirement) error {
}
switch fc := c.Args[0].(type) {
case *VarRef, *Wildcard:
case *VarRef, *Wildcard, *RegexLiteral:
// do nothing
case *Call:
if fc.Name != "distinct" || expr.Name != "count" {
@ -1772,7 +1793,7 @@ func (s *SelectStatement) validateAggregates(tr targetRequirement) error {
return fmt.Errorf("invalid number of arguments for %s, expected %d, got %d", expr.Name, exp, got)
}
switch fc := expr.Args[0].(type) {
case *VarRef, *Wildcard:
case *VarRef, *Wildcard, *RegexLiteral:
// do nothing
case *Call:
if fc.Name != "distinct" || expr.Name != "count" {

17
influxql/ast_test.go
View File

@ -390,6 +390,23 @@ func TestSelectStatement_RewriteFields(t *testing.T) {
stmt: `SELECT mean(*) AS alias FROM cpu`,
rewrite: `SELECT mean(value1::float) AS alias_value1, mean(value2::integer) AS alias_value2 FROM cpu`,
},
// Query regex
{
stmt: `SELECT /1/ FROM cpu`,
rewrite: `SELECT value1::float FROM cpu`,
},
{
stmt: `SELECT value1 FROM cpu GROUP BY /h/`,
rewrite: `SELECT value1::float FROM cpu GROUP BY host`,
},
// Query regex
{
stmt: `SELECT mean(/1/) FROM cpu`,
rewrite: `SELECT mean(value1::float) AS mean_value1 FROM cpu`,
},
}
for i, tt := range tests {

82
influxql/parser.go
View File

@ -1915,19 +1915,27 @@ func (p *Parser) parseFields() (Fields, error) {
func (p *Parser) parseField() (*Field, error) {
f := &Field{}
_, pos, _ := p.scanIgnoreWhitespace()
p.unscan()
// Parse the expression first.
expr, err := p.ParseExpr()
// Attempt to parse a regex.
re, err := p.parseRegex()
if err != nil {
return nil, err
} else if re != nil {
f.Expr = re
} else {
_, pos, _ := p.scanIgnoreWhitespace()
p.unscan()
// Parse the expression first.
expr, err := p.ParseExpr()
if err != nil {
return nil, err
}
var c validateField
Walk(&c, expr)
if c.foundInvalid {
return nil, fmt.Errorf("invalid operator %s in SELECT clause at line %d, char %d; operator is intended for WHERE clause", c.badToken, pos.Line+1, pos.Char+1)
}
f.Expr = expr
}
var c validateField
Walk(&c, expr)
if c.foundInvalid {
return nil, fmt.Errorf("invalid operator %s in SELECT clause at line %d, char %d; operator is intended for WHERE clause", c.badToken, pos.Line+1, pos.Char+1)
}
f.Expr = expr
// Parse the alias if the current and next tokens are "WS AS".
alias, err := p.parseAlias()
@ -2115,6 +2123,13 @@ func (p *Parser) parseDimensions() (Dimensions, error) {
// parseDimension parses a single dimension.
func (p *Parser) parseDimension() (*Dimension, error) {
re, err := p.parseRegex()
if err != nil {
return nil, err
} else if re != nil {
return &Dimension{Expr: re}, nil
}
// Parse the expression first.
expr, err := p.ParseExpr()
if err != nil {
@ -2538,27 +2553,50 @@ func (p *Parser) parseRegex() (*RegexLiteral, error) {
// This function assumes the function name and LPAREN have been consumed.
func (p *Parser) parseCall(name string) (*Call, error) {
name = strings.ToLower(name)
// If there's a right paren then just return immediately.
if tok, _, _ := p.scan(); tok == RPAREN {
return &Call{Name: name}, nil
}
p.unscan()
// Otherwise parse function call arguments.
// Parse first function argument if one exists.
var args []Expr
re, err := p.parseRegex()
if err != nil {
return nil, err
} else if re != nil {
args = append(args, re)
} else {
// If there's a right paren then just return immediately.
if tok, _, _ := p.scan(); tok == RPAREN {
return &Call{Name: name}, nil
}
p.unscan()
arg, err := p.ParseExpr()
if err != nil {
return nil, err
}
args = append(args, arg)
}
// Parse additional function arguments if there is a comma.
for {
// If there's not a comma, stop parsing arguments.
if tok, _, _ := p.scanIgnoreWhitespace(); tok != COMMA {
p.unscan()
break
}
re, err := p.parseRegex()
if err != nil {
return nil, err
} else if re != nil {
args = append(args, re)
continue
}
// Parse an expression argument.
arg, err := p.ParseExpr()
if err != nil {
return nil, err
}
args = append(args, arg)
// If there's not a comma next then stop parsing arguments.
if tok, _, _ := p.scan(); tok != COMMA {
p.unscan()
break
}
}
// There should be a right parentheses at the end.