Merge pull request #736 from influxdata/feat/json-query-result-iterator

Implement query.ResultIterator on influxql.Response
pull/10616/head
Michael Desa 2018-08-31 14:39:19 -04:00 committed by GitHub
commit 2717f9396a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 646 additions and 0 deletions

View File

@ -0,0 +1,361 @@
package influxql
import (
"fmt"
"sort"
"strconv"
"time"
"github.com/influxdata/platform/query"
"github.com/influxdata/platform/query/execute"
"github.com/influxdata/platform/query/values"
)
// responseIterator implements query.ResultIterator for a Response.
type responseIterator struct {
response *Response
resultIdx int
}
// NewresponseIterator constructs a responseIterator from a query.ResultIterator.
func NewResponseIterator(r *Response) query.ResultIterator {
return &responseIterator{
response: r,
}
}
// More returns true if there are results left to iterate through.
// It is used to implement query.ResultIterator.
func (r *responseIterator) More() bool {
return r.resultIdx < len(r.response.Results)
}
// Next retrieves the next query.Result.
// It is used to implement query.ResultIterator.
func (r *responseIterator) Next() query.Result {
res := r.response.Results[r.resultIdx]
r.resultIdx++
return newQueryResult(&res)
}
// Cancel is a noop.
// It is used to implement query.ResultIterator.
func (r *responseIterator) Cancel() {}
// Err returns an error if the response contained an error.
// It is used to implement query.ResultIterator.
func (r *responseIterator) Err() error {
if r.response.Err != "" {
return fmt.Errorf(r.response.Err)
}
return nil
}
// seriesIterator is a simple wrapper for Result that implements query.Result and query.TableIterator.
type seriesIterator struct {
result *Result
}
func newQueryResult(r *Result) *seriesIterator {
return &seriesIterator{
result: r,
}
}
// Name returns the results statement id.
// It is used to implement query.Result.
func (r *seriesIterator) Name() string {
return strconv.Itoa(r.result.StatementID)
}
// Tables returns the original as a query.TableIterator.
// It is used to implement query.Result.
func (r *seriesIterator) Tables() query.TableIterator {
return r
}
// Do iterates through the series of a Result.
// It is used to implement query.TableIterator.
func (r *seriesIterator) Do(f func(query.Table) error) error {
for _, row := range r.result.Series {
t, err := newQueryTable(row)
if err != nil {
return err
}
if err := f(t); err != nil {
return err
}
}
return nil
}
// queryTable implements query.Table and query.ColReader.
type queryTable struct {
row *Row
groupKey query.GroupKey
colMeta []query.ColMeta
cols []interface{}
}
func newQueryTable(r *Row) (*queryTable, error) {
t := &queryTable{
row: r,
}
if err := t.translateRowsToColumns(); err != nil {
return nil, err
}
return t, nil
}
// Data in a column is laid out in the following way:
// [ r.row.Columns... , r.tagKeys()... , r.row.Name ]
func (t *queryTable) translateRowsToColumns() error {
cols := t.Cols()
t.cols = make([]interface{}, len(cols))
for i, col := range cols {
switch col.Type {
case query.TFloat:
t.cols[i] = make([]float64, 0, t.Len())
case query.TInt:
t.cols[i] = make([]int64, 0, t.Len())
case query.TUInt:
t.cols[i] = make([]uint64, 0, t.Len())
case query.TString:
t.cols[i] = make([]string, 0, t.Len())
case query.TBool:
t.cols[i] = make([]bool, 0, t.Len())
case query.TTime:
t.cols[i] = make([]values.Time, 0, t.Len())
}
}
for _, els := range t.row.Values {
for i, el := range els {
col := cols[i]
switch col.Type {
case query.TFloat:
val, ok := el.(float64)
if !ok {
return fmt.Errorf("unsupported type %T found in column %s of type %s", val, col.Label, col.Type)
}
t.cols[i] = append(t.cols[i].([]float64), val)
case query.TInt:
val, ok := el.(int64)
if !ok {
return fmt.Errorf("unsupported type %T found in column %s of type %s", val, col.Label, col.Type)
}
t.cols[i] = append(t.cols[i].([]int64), val)
case query.TUInt:
val, ok := el.(uint64)
if !ok {
return fmt.Errorf("unsupported type %T found in column %s of type %s", val, col.Label, col.Type)
}
t.cols[i] = append(t.cols[i].([]uint64), val)
case query.TString:
val, ok := el.(string)
if !ok {
return fmt.Errorf("unsupported type %T found in column %s of type %s", val, col.Label, col.Type)
}
t.cols[i] = append(t.cols[i].([]string), val)
case query.TBool:
val, ok := el.(bool)
if !ok {
return fmt.Errorf("unsupported type %T found in column %s of type %s", val, col.Label, col.Type)
}
t.cols[i] = append(t.cols[i].([]bool), val)
case query.TTime:
switch val := el.(type) {
case int64:
t.cols[i] = append(t.cols[i].([]values.Time), values.Time(val))
case float64:
t.cols[i] = append(t.cols[i].([]values.Time), values.Time(val))
case string:
tm, err := time.Parse(time.RFC3339, val)
if err != nil {
return fmt.Errorf("could not parse string %q as time: %v", val, err)
}
t.cols[i] = append(t.cols[i].([]values.Time), values.ConvertTime(tm))
default:
return fmt.Errorf("unsupported type %T found in column %s", val, col.Label)
}
default:
return fmt.Errorf("invalid type %T found in column %s", el, col.Label)
}
}
j := len(t.row.Columns)
for j < len(t.row.Columns)+len(t.row.Tags) {
col := cols[j]
t.cols[j] = append(t.cols[j].([]string), t.row.Tags[col.Label])
j++
}
t.cols[j] = append(t.cols[j].([]string), t.row.Name)
}
return nil
}
// Key constructs the query.GroupKey for a Row from the rows
// tags and measurement.
// It is used to implement query.Table and query.ColReader.
func (r *queryTable) Key() query.GroupKey {
if r.groupKey == nil {
cols := make([]query.ColMeta, len(r.row.Tags)+1) // plus one is for measurement
vs := make([]values.Value, len(r.row.Tags)+1)
kvs := make([]interface{}, len(r.row.Tags)+1)
colMeta := r.Cols()
labels := append(r.tagKeys(), "_measurement")
for j, label := range labels {
idx := execute.ColIdx(label, colMeta)
if idx < 0 {
panic(fmt.Errorf("table invalid: missing group column %q", label))
}
cols[j] = colMeta[idx]
kvs[j] = "string"
v, err := values.NewValue(kvs[j], execute.ConvertToKind(cols[j].Type))
if err != nil {
panic(err)
}
vs[j] = v
}
r.groupKey = execute.NewGroupKey(cols, vs)
}
return r.groupKey
}
// tags returns the tag keys for a Row.
func (r *queryTable) tagKeys() []string {
tags := []string{}
for t := range r.row.Tags {
tags = append(tags, t)
}
sort.Strings(tags)
return tags
}
// Cols returns the columns for a row where the data is laid out in the following way:
// [ r.row.Columns... , r.tagKeys()... , r.row.Name ]
// It is used to implement query.Table and query.ColReader.
func (r *queryTable) Cols() []query.ColMeta {
if r.colMeta == nil {
colMeta := make([]query.ColMeta, len(r.row.Columns)+len(r.row.Tags)+1)
for i, col := range r.row.Columns {
colMeta[i] = query.ColMeta{
Label: col,
Type: query.TInvalid,
}
if col == "time" {
// rename the time column
colMeta[i].Label = "_time"
colMeta[i].Type = query.TTime
}
}
if len(r.row.Values) < 1 {
panic("must have at least one value")
}
data := r.row.Values[0]
for i := range r.row.Columns {
v := data[i]
if colMeta[i].Label == "_time" {
continue
}
switch v.(type) {
case float64:
colMeta[i].Type = query.TFloat
case int64:
colMeta[i].Type = query.TInt
case uint64:
colMeta[i].Type = query.TUInt
case bool:
colMeta[i].Type = query.TBool
case string:
colMeta[i].Type = query.TString
}
}
tags := r.tagKeys()
leng := len(r.row.Columns)
for i, tag := range tags {
colMeta[leng+i] = query.ColMeta{
Label: tag,
Type: query.TString,
}
}
leng = leng + len(tags)
colMeta[leng] = query.ColMeta{
Label: "_measurement",
Type: query.TString,
}
r.colMeta = colMeta
}
return r.colMeta
}
// Do applies f to itself. This is because Row is a query.ColReader.
// It is used to implement query.Table.
func (r *queryTable) Do(f func(query.ColReader) error) error {
return f(r)
}
// RefCount is a noop.
// It is used to implement query.ColReader.
func (r *queryTable) RefCount(n int) {}
// Empty returns true if a Row has no values.
// It is used to implement query.Table.
func (r *queryTable) Empty() bool { return r.Len() == 0 }
// Len returns the length or r.row.Values
// It is used to implement query.ColReader.
func (r *queryTable) Len() int {
return len(r.row.Values)
}
// Bools returns the values in column index j as bools.
// It will panic if the column is not a []bool.
// It is used to implement query.ColReader.
func (r *queryTable) Bools(j int) []bool {
return r.cols[j].([]bool)
}
// Ints returns the values in column index j as ints.
// It will panic if the column is not a []int64.
// It is used to implement query.ColReader.
func (r *queryTable) Ints(j int) []int64 {
return r.cols[j].([]int64)
}
// UInts returns the values in column index j as ints.
// It will panic if the column is not a []uint64.
// It is used to implement query.ColReader.
func (r *queryTable) UInts(j int) []uint64 {
return r.cols[j].([]uint64)
}
// Floats returns the values in column index j as floats.
// It will panic if the column is not a []float64.
// It is used to implement query.ColReader.
func (r *queryTable) Floats(j int) []float64 {
return r.cols[j].([]float64)
}
// Strings returns the values in column index j as strings.
// It will panic if the column is not a []string.
// It is used to implement query.ColReader.
func (r *queryTable) Strings(j int) []string {
return r.cols[j].([]string)
}
// Times returns the values in column index j as values.Times.
// It will panic if the column is not a []values.Time.
// It is used to implement query.ColReader.
func (r *queryTable) Times(j int) []values.Time {
return r.cols[j].([]values.Time)
}

View File

@ -0,0 +1,285 @@
package influxql_test
import (
"bytes"
"regexp"
"testing"
"github.com/andreyvit/diff"
"github.com/google/go-cmp/cmp"
"github.com/influxdata/platform/query/csv"
"github.com/influxdata/platform/query/influxql"
)
var crlfPattern = regexp.MustCompile(`\r?\n`)
func toCRLF(data string) []byte {
return []byte(crlfPattern.ReplaceAllString(data, "\r\n"))
}
func TestResponse_ResultIterator(t *testing.T) {
type testCase struct {
name string
encoded []byte
response *influxql.Response
err error
}
tests := []testCase{
{
name: "single series",
encoded: toCRLF(`#datatype,string,long,dateTime:RFC3339,double,long,string,boolean,string,string,string
#group,false,false,false,false,false,false,false,true,true,true
#default,0,,,,,,,,,
,result,table,_time,usage_user,test,mystr,this,cpu,host,_measurement
,,0,2018-08-29T13:08:47Z,10.2,10,yay,true,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,12.1,20,nay,false,cpu-total,a,cpu
,,0,2018-08-29T13:08:47Z,112,30,way,false,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,123.2,40,pay,true,cpu-total,a,cpu
`),
response: &influxql.Response{
Results: []influxql.Result{
{
StatementID: 0,
Series: []*influxql.Row{
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "a",
},
Columns: []string{"time", "usage_user", "test", "mystr", "this"},
Values: [][]interface{}{
{int64(1535548127000000000), 10.2, int64(10), "yay", true},
{int64(1535548128000000000), 12.1, int64(20), "nay", false},
{int64(1535548127000000000), 112.0, int64(30), "way", false},
{int64(1535548128000000000), 123.2, int64(40), "pay", true},
},
},
},
},
},
},
},
{
name: "multiple series",
encoded: toCRLF(`#datatype,string,long,dateTime:RFC3339,double,long,string,string,string,string
#group,false,false,false,false,false,false,true,true,true
#default,0,,,,,,,,
,result,table,_time,usage_user,test,mystr,cpu,host,_measurement
,,0,2018-08-29T13:08:47Z,10.2,10,yay,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,12.1,20,nay,cpu-total,a,cpu
,,0,2018-08-29T13:08:47Z,112,30,way,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,123.2,40,pay,cpu-total,a,cpu
,,1,2018-08-29T18:27:31Z,10.2,10,yay,cpu-total,b,cpu
,,1,2018-08-29T18:27:31Z,12.1,20,nay,cpu-total,b,cpu
,,1,2018-08-29T18:27:31Z,112,30,way,cpu-total,b,cpu
,,1,2018-08-29T18:27:31Z,123.2,40,pay,cpu-total,b,cpu
`),
response: &influxql.Response{
Results: []influxql.Result{
{
StatementID: 0,
Series: []*influxql.Row{
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "a",
},
Columns: []string{"time", "usage_user", "test", "mystr"},
Values: [][]interface{}{
{float64(1535548127000000000), 10.2, int64(10), "yay"},
{float64(1535548128000000000), 12.1, int64(20), "nay"},
{float64(1535548127000000000), 112.0, int64(30), "way"},
{float64(1535548128000000000), 123.2, int64(40), "pay"},
},
},
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "b",
},
Columns: []string{"time", "usage_user", "test", "mystr"},
Values: [][]interface{}{
{"2018-08-29T18:27:31Z", 10.2, int64(10), "yay"},
{"2018-08-29T18:27:31Z", 12.1, int64(20), "nay"},
{"2018-08-29T18:27:31Z", 112.0, int64(30), "way"},
{"2018-08-29T18:27:31Z", 123.2, int64(40), "pay"},
},
},
},
},
},
},
},
{
name: "multiple series with same columns but different types",
encoded: toCRLF(`#datatype,string,long,dateTime:RFC3339,double,long,string,string,string,string
#group,false,false,false,false,false,false,true,true,true
#default,0,,,,,,,,
,result,table,_time,usage_user,test,mystr,cpu,host,_measurement
,,0,2018-08-29T13:08:47Z,10.2,1,yay,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,12.1,2,nay,cpu-total,a,cpu
,,0,2018-08-29T13:08:47Z,112,3,way,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,123.2,4,pay,cpu-total,a,cpu
#datatype,string,long,dateTime:RFC3339,double,double,string,string,string,string
#group,false,false,false,false,false,false,true,true,true
#default,0,,,,,,,,
,result,table,_time,usage_user,test,mystr,cpu,host,_measurement
,,1,2018-08-29T13:08:47Z,10.2,10,yay,cpu-total,a,cpu
,,1,2018-08-29T13:08:48Z,12.1,20,nay,cpu-total,a,cpu
,,1,2018-08-29T13:08:47Z,112,30,way,cpu-total,a,cpu
,,1,2018-08-29T13:08:48Z,123.2,40,pay,cpu-total,a,cpu
`),
response: &influxql.Response{
Results: []influxql.Result{
{
StatementID: 0,
Series: []*influxql.Row{
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "a",
},
Columns: []string{"time", "usage_user", "test", "mystr"},
Values: [][]interface{}{
{int64(1535548127000000000), 10.2, int64(1), "yay"},
{int64(1535548128000000000), 12.1, int64(2), "nay"},
{int64(1535548127000000000), 112.0, int64(3), "way"},
{int64(1535548128000000000), 123.2, int64(4), "pay"},
},
},
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "a",
},
Columns: []string{"time", "usage_user", "test", "mystr"},
Values: [][]interface{}{
{int64(1535548127000000000), 10.2, float64(10), "yay"},
{int64(1535548128000000000), 12.1, float64(20), "nay"},
{int64(1535548127000000000), 112.0, float64(30), "way"},
{int64(1535548128000000000), 123.2, float64(40), "pay"},
},
},
},
},
},
},
},
{
name: "multiple results",
encoded: toCRLF(`#datatype,string,long,dateTime:RFC3339,double,long,string,string,string,string
#group,false,false,false,false,false,false,true,true,true
#default,0,,,,,,,,
,result,table,_time,usage_user,test,mystr,cpu,host,_measurement
,,0,2018-08-29T13:08:47Z,10.2,10,yay,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,12.1,20,nay,cpu-total,a,cpu
,,0,2018-08-29T13:08:47Z,112,30,way,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,123.2,40,pay,cpu-total,a,cpu
,,1,2018-08-29T13:08:47Z,10.2,10,yay,cpu-total,b,cpu
,,1,2018-08-29T13:08:48Z,12.1,20,nay,cpu-total,b,cpu
,,1,2018-08-29T13:08:47Z,112,30,way,cpu-total,b,cpu
,,1,2018-08-29T13:08:48Z,123.2,40,pay,cpu-total,b,cpu
#datatype,string,long,dateTime:RFC3339,double,long,string,string,string,string
#group,false,false,false,false,false,false,true,true,true
#default,1,,,,,,,,
,result,table,_time,usage_user,test,mystr,cpu,host,_measurement
,,0,2018-08-29T13:08:47Z,10.2,10,yay,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,12.1,20,nay,cpu-total,a,cpu
,,0,2018-08-29T13:08:47Z,112,30,way,cpu-total,a,cpu
,,0,2018-08-29T13:08:48Z,123.2,40,pay,cpu-total,a,cpu
`),
response: &influxql.Response{
Results: []influxql.Result{
{
StatementID: 0,
Series: []*influxql.Row{
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "a",
},
Columns: []string{"time", "usage_user", "test", "mystr"},
Values: [][]interface{}{
{int64(1535548127000000000), 10.2, int64(10), "yay"},
{int64(1535548128000000000), 12.1, int64(20), "nay"},
{int64(1535548127000000000), 112.0, int64(30), "way"},
{int64(1535548128000000000), 123.2, int64(40), "pay"},
},
},
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "b",
},
Columns: []string{"time", "usage_user", "test", "mystr"},
Values: [][]interface{}{
{int64(1535548127000000000), 10.2, int64(10), "yay"},
{int64(1535548128000000000), 12.1, int64(20), "nay"},
{int64(1535548127000000000), 112.0, int64(30), "way"},
{int64(1535548128000000000), 123.2, int64(40), "pay"},
},
},
},
},
{
StatementID: 1,
Series: []*influxql.Row{
{
Name: "cpu",
Tags: map[string]string{
"cpu": "cpu-total",
"host": "a",
},
Columns: []string{"time", "usage_user", "test", "mystr"},
Values: [][]interface{}{
{int64(1535548127000000000), 10.2, int64(10), "yay"},
{int64(1535548128000000000), 12.1, int64(20), "nay"},
{int64(1535548127000000000), 112.0, int64(30), "way"},
{int64(1535548128000000000), 123.2, int64(40), "pay"},
},
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
encoderConfig := csv.DefaultEncoderConfig()
encoder := csv.NewMultiResultEncoder(encoderConfig)
var got bytes.Buffer
n, err := encoder.Encode(&got, influxql.NewResponseIterator(tt.response))
if err != nil && tt.err != nil {
if err.Error() != tt.err.Error() {
t.Errorf("unexpected error want: %s\n got: %s\n", tt.err.Error(), err.Error())
}
} else if err != nil {
t.Errorf("unexpected error want: none\n got: %s\n", err.Error())
} else if tt.err != nil {
t.Errorf("unexpected error want: %s\n got: none", tt.err.Error())
}
if g, w := got.String(), string(tt.encoded); g != w {
t.Errorf("unexpected encoding -want/+got:\n%s", diff.LineDiff(w, g))
}
if g, w := n, int64(len(tt.encoded)); g != w {
t.Errorf("unexpected encoding count -want/+got:\n%s", cmp.Diff(w, g))
}
})
}
}