Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Implement MEASUREMENT cardinality estimation

pull/8984/head
Edd Robinson 2017-10-17 15:34:27 +01:00
parent 3079b41f00
commit f80591bfa1
8 changed files with 144 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
coordinator/statement_executor.go
View File

@ -174,6 +174,8 @@ func (e *StatementExecutor) ExecuteStatement(stmt influxql.Statement, ctx query.
rows, err = e.executeShowGrantsForUserStatement(stmt)
case *influxql.ShowMeasurementsStatement:
return e.executeShowMeasurementsStatement(stmt, &ctx)
case *influxql.ShowMeasurementCardinalityStatement:
rows, err = e.executeShowMeasurementCardinalityStatement(stmt)
case *influxql.ShowRetentionPoliciesStatement:
rows, err = e.executeShowRetentionPoliciesStatement(stmt)
case *influxql.ShowSeriesCardinalityStatement:
@ -781,6 +783,18 @@ func (e *StatementExecutor) executeShowMeasurementsStatement(q *influxql.ShowMea
})
}
func (e *StatementExecutor) executeShowMeasurementCardinalityStatement(stmt *influxql.ShowMeasurementCardinalityStatement) (models.Rows, error) {
n, err := e.TSDBStore.MeasurementsCardinality(stmt.Database)
if err != nil {
return nil, err
}
return []*models.Row{&models.Row{
Columns: []string{"cardinality"},
Values: [][]interface{}{{n}},
}}, nil
}
func (e *StatementExecutor) executeShowRetentionPoliciesStatement(q *influxql.ShowRetentionPoliciesStatement) (models.Rows, error) {
if q.Database == "" {
return nil, ErrDatabaseNameRequired
@ -1165,6 +1179,10 @@ func (e *StatementExecutor) NormalizeStatement(stmt influxql.Statement, defaultD
if node.Database == "" {
node.Database = defaultDatabase
}
case *influxql.ShowMeasurementCardinalityStatement:
if node.Database == "" {
node.Database = defaultDatabase
}
case *influxql.Measurement:
switch stmt.(type) {
case *influxql.DropSeriesStatement, *influxql.DeleteSeriesStatement:
@ -1237,6 +1255,7 @@ type TSDBStore interface {
TagValues(auth query.Authorizer, database string, cond influxql.Expr) ([]tsdb.TagValues, error)
SeriesCardinality(database string) (int64, error)
MeasurementsCardinality(database string) (int64, error)
}
var _ TSDBStore = LocalTSDBStore{}

19
coordinator/statement_executor_test.go
View File

@ -319,13 +319,14 @@ type TSDBStore struct {
RestoreShardFn func(id uint64, r io.Reader) error
BackupShardFn func(id uint64, since time.Time, w io.Writer) error
DeleteDatabaseFn func(name string) error
DeleteMeasurementFn func(database, name string) error
DeleteRetentionPolicyFn func(database, name string) error
DeleteShardFn func(id uint64) error
DeleteSeriesFn func(database string, sources []influxql.Source, condition influxql.Expr) error
ShardGroupFn func(ids []uint64) tsdb.ShardGroup
SeriesCardinalityFn func(database string) (int64, error)
DeleteDatabaseFn func(name string) error
DeleteMeasurementFn func(database, name string) error
DeleteRetentionPolicyFn func(database, name string) error
DeleteShardFn func(id uint64) error
DeleteSeriesFn func(database string, sources []influxql.Source, condition influxql.Expr) error
ShardGroupFn func(ids []uint64) tsdb.ShardGroup
MeasurementsCardinalityFn func(database string) (int64, error)
SeriesCardinalityFn func(database string) (int64, error)
}
func (s *TSDBStore) CreateShard(database, policy string, shardID uint64, enabled bool) error {
@ -379,6 +380,10 @@ func (s *TSDBStore) MeasurementNames(database string, cond influxql.Expr) ([][]b
return nil, nil
}
func (s *TSDBStore) MeasurementsCardinality(database string) (int64, error) {
return s.MeasurementsCardinalityFn(database)
}
func (s *TSDBStore) TagValues(_ query.Authorizer, database string, cond influxql.Expr) ([]tsdb.TagValues, error) {
return nil, nil
}

17
influxql/ast.go
View File

@ -2320,6 +2320,7 @@ func (s *DropContinuousQueryStatement) DefaultDatabase() string {
// ShowMeasurementCardinalityStatement represents a command for listing measurement cardinality.
type ShowMeasurementCardinalityStatement struct {
Exact bool // If false then cardinality estimation will be used.
Database string
Sources Sources
Condition Expr
@ -2330,7 +2331,18 @@ type ShowMeasurementCardinalityStatement struct {
// String returns a string representation of the statement.
func (s *ShowMeasurementCardinalityStatement) String() string {
var buf bytes.Buffer
_, _ = buf.WriteString("SHOW MEASUREMENT CARDINALITY")
_, _ = buf.WriteString("SHOW MEASUREMENT")
if !s.Exact {
_, _ = buf.WriteString(" CARDINALITY")
if s.Database != "" {
_, _ = buf.WriteString(" ON ")
_, _ = buf.WriteString(QuoteIdent(s.Database))
}
return buf.String()
}
_, _ = buf.WriteString(" EXACT CARDINALITY")
if s.Database != "" {
_, _ = buf.WriteString(" ON ")
@ -2360,6 +2372,9 @@ func (s *ShowMeasurementCardinalityStatement) String() string {
// RequiredPrivileges returns the privilege required to execute a ShowMeasurementCardinalityStatement.
func (s *ShowMeasurementCardinalityStatement) RequiredPrivileges() (ExecutionPrivileges, error) {
if !s.Exact {
return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil
}
return s.Sources.RequiredPrivileges()
}

5
influxql/parse_tree.go
View File

@ -127,8 +127,11 @@ func init() {
show.Group(GRANTS).Handle(FOR, func(p *Parser) (Statement, error) {
return p.parseGrantsForUserStatement()
})
show.Group(MEASUREMENT).Handle(EXACT, func(p *Parser) (Statement, error) {
return p.parseShowMeasurementCardinalityStatement(true)
})
show.Group(MEASUREMENT).Handle(CARDINALITY, func(p *Parser) (Statement, error) {
return p.parseShowMeasurementCardinalityStatement()
return p.parseShowMeasurementCardinalityStatement(false)
})
show.Handle(MEASUREMENTS, func(p *Parser) (Statement, error) {
return p.parseShowMeasurementsStatement()

19
influxql/parser.go
View File

@ -1008,9 +1008,17 @@ func (p *Parser) parseShowSeriesCardinalityStatement(exact bool) (Statement, err
return stmt, nil
}
// This function assumes the "SHOW MEASUREMENT CARDINALITY" tokens have already been consumed.
func (p *Parser) parseShowMeasurementCardinalityStatement() (Statement, error) {
stmt := &ShowMeasurementCardinalityStatement{}
// This function assumes the "SHOW MEASUREMENT EXACT" or "SHOW MEASUREMENT CARDINALITY"
// tokens have already been consumed.
func (p *Parser) parseShowMeasurementCardinalityStatement(exact bool) (Statement, error) {
stmt := &ShowMeasurementCardinalityStatement{Exact: exact}
if stmt.Exact {
// Parse remaining CARDINALITY token
if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != CARDINALITY {
return nil, newParseError(tokstr(tok, lit), []string{"CARDINALITY"}, pos)
}
}
// Parse optional ON clause.
var err error
@ -1022,6 +1030,11 @@ func (p *Parser) parseShowMeasurementCardinalityStatement() (Statement, error) {
p.Unscan()
}
// Estimation command doesn't support any further versions of the command.
if !stmt.Exact {
return stmt, nil
}
// Parse optional FROM.
if tok, _, _ := p.ScanIgnoreWhitespace(); tok == FROM {
if stmt.Sources, err = p.parseSources(false); err != nil {

73
influxql/parser_test.go
View File

@ -1690,26 +1690,46 @@ func TestParser_ParseStatement(t *testing.T) {
stmt: &influxql.ShowMeasurementCardinalityStatement{},
},
// SHOW MEASUREMENT CARDINALITY FROM cpu
{
s: `SHOW MEASUREMENT CARDINALITY FROM cpu`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
},
},
// SHOW MEASUREMENT CARDINALITY ON db0
// SHOW MEASUREMENT CARDINALITY ON db0 statement
{
s: `SHOW MEASUREMENT CARDINALITY ON db0`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: false,
Database: "db0",
},
},
// SHOW MEASUREMENT CARDINALITY FROM /<regex>/
// SHOW MEASUREMENT EXACT CARDINALITY statement
{
s: `SHOW MEASUREMENT CARDINALITY FROM /[cg]pu/`,
s: `SHOW MEASUREMENT EXACT CARDINALITY`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: true,
},
},
// SHOW MEASUREMENT EXACT CARDINALITY FROM cpu
{
s: `SHOW MEASUREMENT EXACT CARDINALITY FROM cpu`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: true,
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
},
},
// SHOW MEASUREMENT EXACT CARDINALITY ON db0
{
s: `SHOW MEASUREMENT EXACT CARDINALITY ON db0`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: true,
Database: "db0",
},
},
// SHOW MEASUREMENT EXACT CARDINALITY FROM /<regex>/
{
s: `SHOW MEASUREMENT EXACT CARDINALITY FROM /[cg]pu/`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: true,
Sources: []influxql.Source{
&influxql.Measurement{
Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`[cg]pu`)},
@ -1718,22 +1738,25 @@ func TestParser_ParseStatement(t *testing.T) {
},
},
// SHOW MEASUREMENT CARDINALITY with OFFSET 0
// SHOW MEASUREMENT EXACT CARDINALITY with OFFSET 0
{
s: `SHOW MEASUREMENT CARDINALITY OFFSET 0`,
stmt: &influxql.ShowMeasurementCardinalityStatement{Offset: 0},
},
// SHOW MEASUREMENT CARDINALITY with LIMIT 2 OFFSET 0
{
s: `SHOW MEASUREMENT CARDINALITY LIMIT 2 OFFSET 0`,
stmt: &influxql.ShowMeasurementCardinalityStatement{Offset: 0, Limit: 2},
},
// SHOW MEASUREMENT CARDINALITY WHERE with ORDER BY and LIMIT
{
s: `SHOW MEASUREMENT CARDINALITY WHERE region = 'order by desc' LIMIT 10`,
s: `SHOW MEASUREMENT EXACT CARDINALITY OFFSET 0`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: true, Offset: 0},
},
// SHOW MEASUREMENT EXACT CARDINALITY with LIMIT 2 OFFSET 0
{
s: `SHOW MEASUREMENT EXACT CARDINALITY LIMIT 2 OFFSET 0`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: true, Offset: 0, Limit: 2},
},
// SHOW MEASUREMENT EXACT CARDINALITY WHERE with ORDER BY and LIMIT
{
s: `SHOW MEASUREMENT EXACT CARDINALITY WHERE region = 'order by desc' LIMIT 10`,
stmt: &influxql.ShowMeasurementCardinalityStatement{
Exact: true,
Condition: &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "region"},

6
influxql/statement_rewriter.go
View File

@ -111,9 +111,13 @@ func rewriteShowMeasurementsStatement(stmt *ShowMeasurementsStatement) (Statemen
}
func rewriteShowMeasurementCardinalityStatement(stmt *ShowMeasurementCardinalityStatement) (Statement, error) {
if !stmt.Exact { // Use cardinality estimation and don't rewrite.
return stmt, nil
}
// Check for time in WHERE clause (not supported).
if HasTimeExpr(stmt.Condition) {
return nil, errors.New("SHOW MEASUREMENT CARDINALITY doesn't support time in WHERE clause")
return nil, errors.New("SHOW MEASUREMENT EXACT CARDINALITY doesn't support time in WHERE clause")
}
// Use all measurements, if zero.

37
tests/server_test.go
View File

@ -7462,43 +7462,54 @@ func TestServer_Query_ShowMeasurementCardinality(t *testing.T) {
&Query{
name: `show measurement cardinality`,
command: "SHOW MEASUREMENT CARDINALITY",
exp: `{"results":[{"statement_id":0,"series":[{"columns":["cardinality"],"values":[[3]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show measurement cardinality on db0`,
command: "SHOW MEASUREMENT CARDINALITY ON db0",
exp: `{"results":[{"statement_id":0,"series":[{"columns":["cardinality"],"values":[[3]]}]}]}`,
},
&Query{
name: `show measurement exact cardinality`,
command: "SHOW MEASUREMENT EXACT CARDINALITY",
exp: `{"results":[{"statement_id":0,"series":[{"columns":["count"],"values":[[3]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show measurement cardinality using FROM`,
command: "SHOW MEASUREMENT CARDINALITY FROM cpu",
name: `show measurement exact cardinality using FROM`,
command: "SHOW MEASUREMENT EXACT CARDINALITY FROM cpu",
exp: `{"results":[{"statement_id":0,"series":[{"columns":["count"],"values":[[1]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show measurement cardinality using FROM and regex`,
command: "SHOW MEASUREMENT CARDINALITY FROM /[cg]pu/",
name: `show measurement exact cardinality using FROM and regex`,
command: "SHOW MEASUREMENT EXACT CARDINALITY FROM /[cg]pu/",
exp: `{"results":[{"statement_id":0,"series":[{"columns":["count"],"values":[[2]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show measurement cardinality using FROM and regex - no matches`,
command: "SHOW MEASUREMENT CARDINALITY FROM /.*zzzzz.*/",
name: `show measurement exact cardinality using FROM and regex - no matches`,
command: "SHOW MEASUREMENT EXACT CARDINALITY FROM /.*zzzzz.*/",
exp: `{"results":[{"statement_id":0}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show measurement cardinality where tag matches regular expression`,
command: "SHOW MEASUREMENT CARDINALITY WHERE region =~ /ca.*/",
name: `show measurement exact cardinality where tag matches regular expression`,
command: "SHOW MEASUREMENT EXACT CARDINALITY WHERE region =~ /ca.*/",
exp: `{"results":[{"statement_id":0,"series":[{"columns":["count"],"values":[[2]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show measurement cardinality where tag does not match a regular expression`,
command: "SHOW MEASUREMENT CARDINALITY WHERE region !~ /ca.*/",
name: `show measurement exact cardinality where tag does not match a regular expression`,
command: "SHOW MEASUREMENT EXACT CARDINALITY WHERE region !~ /ca.*/",
exp: `{"results":[{"statement_id":0,"series":[{"columns":["count"],"values":[[2]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show measurement cardinality with time in WHERE clauses errors`,
command: `SHOW MEASUREMENT CARDINALITY WHERE time > now() - 1h`,
exp: `{"results":[{"statement_id":0,"error":"SHOW MEASUREMENT CARDINALITY doesn't support time in WHERE clause"}]}`,
name: `show measurement exact cardinality with time in WHERE clauses errors`,
command: `SHOW MEASUREMENT EXACT CARDINALITY WHERE time > now() - 1h`,
exp: `{"results":[{"statement_id":0,"error":"SHOW MEASUREMENT EXACT CARDINALITY doesn't support time in WHERE clause"}]}`,
params: url.Values{"db": []string{"db0"}},
},
}...)