From a59e6b8d257eaff954129796c354641dd0fb5192 Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Wed, 9 Jan 2019 12:33:29 -0600 Subject: [PATCH] refactor: rename DoArrow to Do (#2372) See influxdata/flux#783 for details. --- go.mod | 8 ++-- go.sum | 19 +++++---- query/functions/outputs/to.go | 12 +++--- query/influxql/response_iterator.go | 6 --- query/influxql/result.go | 2 +- storage/reads/table.gen.go | 60 ----------------------------- storage/reads/table.gen.go.tmpl | 12 ------ storage/reads/table.go | 12 ------ task/backend/executor/executor.go | 2 +- task/backend/query_logreader.go | 8 ++-- 10 files changed, 28 insertions(+), 113 deletions(-) diff --git a/go.mod b/go.mod index 5fd77915be..8813b519b1 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/SAP/go-hdb v0.13.1 // indirect github.com/SermoDigital/jose v0.9.1 // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 - github.com/apache/arrow/go/arrow v0.0.0-20181217213538-e9ed591db9cb + github.com/apache/arrow/go/arrow v0.0.0-20190107214733-134081bea48d github.com/apex/log v1.1.0 // indirect github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect github.com/armon/go-radix v1.0.0 // indirect @@ -70,8 +70,10 @@ require ( github.com/hashicorp/go-version v1.1.0 // indirect github.com/hashicorp/raft v1.0.0 // indirect github.com/hashicorp/vault v0.11.5 - github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181219175933-9dbe04db0e34 // indirect - github.com/influxdata/flux v0.13.0 + github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181106190520-2236f141171e // indirect + github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/influxdata/flux v0.13.1-0.20190109181605-45cc7b741a21 github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 github.com/jefferai/jsonx v0.0.0-20160721235117-9cc31c3135ee // indirect diff --git a/go.sum b/go.sum index 16ec0f647a..6e23b145a2 100644 --- a/go.sum +++ b/go.sum @@ -36,10 +36,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= -github.com/apache/arrow/go/arrow v0.0.0-20181031164735-a56c009257a7 h1:+leX3wRmpUQHMlRdNw982F5GJ7ty683B4x9EdT7iPVs= -github.com/apache/arrow/go/arrow v0.0.0-20181031164735-a56c009257a7/go.mod h1:GjvccvtI06FGFvRU1In/maF7tKp3h7GBV9Sexo5rNPM= -github.com/apache/arrow/go/arrow v0.0.0-20181217213538-e9ed591db9cb h1:p6xQwsjxRtuIrUDjGAFuro04BO0GNJ9V2troYRY8kmQ= -github.com/apache/arrow/go/arrow v0.0.0-20181217213538-e9ed591db9cb/go.mod h1:GjvccvtI06FGFvRU1In/maF7tKp3h7GBV9Sexo5rNPM= +github.com/apache/arrow/go/arrow v0.0.0-20190107214733-134081bea48d h1:YeyuRFfB3L0OeTishYaVDXN8ZjsmW//+xLkZ7N95nVg= +github.com/apache/arrow/go/arrow v0.0.0-20190107214733-134081bea48d/go.mod h1:GjvccvtI06FGFvRU1In/maF7tKp3h7GBV9Sexo5rNPM= github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI= github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= @@ -79,6 +77,7 @@ github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882b github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -194,8 +193,8 @@ github.com/hashicorp/raft v1.0.0/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcb github.com/hashicorp/vault v0.11.5 h1:6G3922BuHAxy3icIgSTJiv6GQCqFgdmXBvn3L9bNrZA= github.com/hashicorp/vault v0.11.5/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0= github.com/hashicorp/vault v0.11.5/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0= -github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181219175933-9dbe04db0e34 h1:ErMAmfTAS2ZTJ14uKNMu/SjymcjT1mz6dji1b9s1Ftw= -github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181219175933-9dbe04db0e34/go.mod h1:VJHHT2SC1tAPrfENQeBhLlb5FbZoKZM+oC/ROmEftz0= +github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181106190520-2236f141171e h1:2Hwd2Yi0/qjAC6ujOu6WBVXAak9Snuw0LTYdZkqIdKM= +github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181106190520-2236f141171e/go.mod h1:VJHHT2SC1tAPrfENQeBhLlb5FbZoKZM+oC/ROmEftz0= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= @@ -204,16 +203,20 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/imdario/mergo v0.3.4 h1:mKkfHkZWD8dC7WxKx3N9WCF0Y+dLau45704YQmY6H94= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/flux v0.13.0 h1:iWNTxfR8m8LFYNAjSXAYMdxMKHqsBxd3xHlZ7rpl1KA= -github.com/influxdata/flux v0.13.0/go.mod h1:81jeDcHVn1rN5uj9aQ81S72Q8ol8If7N0zM0G8TnxTE= +github.com/influxdata/flux v0.13.1-0.20190109181605-45cc7b741a21 h1:zeeKoL71vc9yy99h8wFEIU0RFLefzAl4sPm0kY0XZvQ= +github.com/influxdata/flux v0.13.1-0.20190109181605-45cc7b741a21/go.mod h1:0f5Yrm4VPSd/Ne6jIVOVtPo0MFe6jpLCr6vdaZYp7wY= github.com/influxdata/goreleaser v0.86.2-0.20181010170531-0fd209ba67f5/go.mod h1:aVuBpDAT5VtjtUxzvBt8HOd0buzvvk7OX3H2iaviixg= github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 h1:CFx+pP90q/qg3spoiZjf8donE4WpAdjeJfPOcoNqkWo= github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= diff --git a/query/functions/outputs/to.go b/query/functions/outputs/to.go index e933f99f91..6de5ba9722 100644 --- a/query/functions/outputs/to.go +++ b/query/functions/outputs/to.go @@ -490,7 +490,7 @@ func writeTable(t *ToTransformation, tbl flux.Table) error { measurementStats := make(map[string]Stats) measurementName := "" - return tbl.DoArrow(func(er flux.ArrowColReader) error { + return tbl.Do(func(er flux.ColReader) error { var pointTime time.Time var points models.Points var tags models.Tags @@ -504,7 +504,7 @@ func writeTable(t *ToTransformation, tbl flux.Table) error { case col.Label == spec.MeasurementColumn: measurementName = string(er.Strings(j).Value(i)) case col.Label == timeColLabel: - pointTime = execute.ValueForRowArrow(er, i, j).Time().Time() + pointTime = execute.ValueForRow(er, i, j).Time().Time() case isTag[j]: if col.Type != flux.TString { return errors.New("invalid type for tag column") @@ -567,7 +567,7 @@ func writeTable(t *ToTransformation, tbl flux.Table) error { return err } points = append(points, pt) - if err := execute.AppendRecordArrow(i, er, builder); err != nil { + if err := execute.AppendRecord(i, er, builder); err != nil { return err } } @@ -576,7 +576,7 @@ func writeTable(t *ToTransformation, tbl flux.Table) error { }) } -func defaultFieldMapping(er flux.ArrowColReader, row int) (values.Object, error) { +func defaultFieldMapping(er flux.ColReader, row int) (values.Object, error) { fieldColumnIdx := execute.ColIdx(defaultFieldColLabel, er.Cols()) valueColumnIdx := execute.ColIdx(execute.DefaultValueColLabel, er.Cols()) @@ -588,10 +588,10 @@ func defaultFieldMapping(er flux.ArrowColReader, row int) (values.Object, error) return nil, errors.New("table has no _value column") } - value := execute.ValueForRowArrow(er, row, valueColumnIdx) + value := execute.ValueForRow(er, row, valueColumnIdx) fieldValueMapping := values.NewObject() - field := execute.ValueForRowArrow(er, row, fieldColumnIdx) + field := execute.ValueForRow(er, row, fieldColumnIdx) fieldValueMapping.Set(field.Str(), value) return fieldValueMapping, nil diff --git a/query/influxql/response_iterator.go b/query/influxql/response_iterator.go index c9a36a4e82..37a1762e56 100644 --- a/query/influxql/response_iterator.go +++ b/query/influxql/response_iterator.go @@ -341,12 +341,6 @@ func (r *queryTable) Cols() []flux.ColMeta { // Do applies f to itself. This is because Row is a flux.ColReader. // It is used to implement flux.Table. func (r *queryTable) Do(f func(flux.ColReader) error) error { - return r.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (r *queryTable) DoArrow(f func(flux.ArrowColReader) error) error { return f(r) } diff --git a/query/influxql/result.go b/query/influxql/result.go index cd6d8ead4c..9297c04247 100644 --- a/query/influxql/result.go +++ b/query/influxql/result.go @@ -97,7 +97,7 @@ func (e *MultiResultEncoder) Encode(w io.Writer, results flux.ResultIterator) (i row.Columns[v] = k } - if err := tbl.DoArrow(func(cr flux.ArrowColReader) error { + if err := tbl.Do(func(cr flux.ColReader) error { // Preallocate the number of rows for the response to make this section // of code easier to read. Find a time column which should exist // in the output. diff --git a/storage/reads/table.gen.go b/storage/reads/table.gen.go index e9b3ef404b..326c367559 100644 --- a/storage/reads/table.gen.go +++ b/storage/reads/table.gen.go @@ -73,12 +73,6 @@ func (t *floatTable) Statistics() flux.Statistics { } func (t *floatTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *floatTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -168,12 +162,6 @@ func (t *floatGroupTable) Close() { } func (t *floatGroupTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *floatGroupTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -311,12 +299,6 @@ func (t *integerTable) Statistics() flux.Statistics { } func (t *integerTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *integerTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -406,12 +388,6 @@ func (t *integerGroupTable) Close() { } func (t *integerGroupTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *integerGroupTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -549,12 +525,6 @@ func (t *unsignedTable) Statistics() flux.Statistics { } func (t *unsignedTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *unsignedTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -644,12 +614,6 @@ func (t *unsignedGroupTable) Close() { } func (t *unsignedGroupTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *unsignedGroupTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -787,12 +751,6 @@ func (t *stringTable) Statistics() flux.Statistics { } func (t *stringTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *stringTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -882,12 +840,6 @@ func (t *stringGroupTable) Close() { } func (t *stringGroupTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *stringGroupTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -1025,12 +977,6 @@ func (t *booleanTable) Statistics() flux.Statistics { } func (t *booleanTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *booleanTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -1120,12 +1066,6 @@ func (t *booleanGroupTable) Close() { } func (t *booleanGroupTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *booleanGroupTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() diff --git a/storage/reads/table.gen.go.tmpl b/storage/reads/table.gen.go.tmpl index c1d01f15ed..47a2558844 100644 --- a/storage/reads/table.gen.go.tmpl +++ b/storage/reads/table.gen.go.tmpl @@ -66,12 +66,6 @@ func (t *{{.name}}Table) Statistics() flux.Statistics { } func (t *{{.name}}Table) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *{{.name}}Table) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() @@ -161,12 +155,6 @@ func (t *{{.name}}GroupTable) Close() { } func (t *{{.name}}GroupTable) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *{{.name}}GroupTable) DoArrow(f func(flux.ArrowColReader) error) error { t.mu.Lock() defer func() { t.closeDone() diff --git a/storage/reads/table.go b/storage/reads/table.go index 0182cb6bb0..a4e4dcfe51 100644 --- a/storage/reads/table.go +++ b/storage/reads/table.go @@ -214,12 +214,6 @@ func (t *tableNoPoints) Close() {} func (t *tableNoPoints) Statistics() flux.Statistics { return flux.Statistics{} } func (t *tableNoPoints) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *tableNoPoints) DoArrow(f func(flux.ArrowColReader) error) error { if t.isCancelled() { return nil } @@ -249,12 +243,6 @@ func newGroupTableNoPoints( func (t *groupTableNoPoints) Close() {} func (t *groupTableNoPoints) Do(f func(flux.ColReader) error) error { - return t.DoArrow(func(cr flux.ArrowColReader) error { - return f(arrow.ColReader(cr)) - }) -} - -func (t *groupTableNoPoints) DoArrow(f func(flux.ArrowColReader) error) error { if t.isCancelled() { return nil } diff --git a/task/backend/executor/executor.go b/task/backend/executor/executor.go index 437fce6930..d70a40c05f 100644 --- a/task/backend/executor/executor.go +++ b/task/backend/executor/executor.go @@ -342,7 +342,7 @@ func (rr *runResult) IsRetryable() bool { return rr.retryable } // exhaustResultIterators drains all the iterators from a flux query Result. func exhaustResultIterators(res flux.Result) error { return res.Tables().Do(func(tbl flux.Table) error { - return tbl.DoArrow(func(flux.ArrowColReader) error { + return tbl.Do(func(flux.ColReader) error { return nil }) }) diff --git a/task/backend/query_logreader.go b/task/backend/query_logreader.go index e50f0feddf..0d57770774 100644 --- a/task/backend/query_logreader.go +++ b/task/backend/query_logreader.go @@ -253,15 +253,15 @@ func (re *runExtractor) Extract(tbl flux.Table) error { switch mv.Str() { case "records": - return tbl.DoArrow(re.extractRecord) + return tbl.Do(re.extractRecord) case "logs": - return tbl.DoArrow(re.extractLog) + return tbl.Do(re.extractLog) default: return fmt.Errorf("unknown measurement: %q", mv.Str()) } } -func (re *runExtractor) extractRecord(cr flux.ArrowColReader) error { +func (re *runExtractor) extractRecord(cr flux.ColReader) error { for i := 0; i < cr.Len(); i++ { var r platform.Run for j, col := range cr.Cols() { @@ -305,7 +305,7 @@ func (re *runExtractor) extractRecord(cr flux.ArrowColReader) error { return nil } -func (re *runExtractor) extractLog(cr flux.ArrowColReader) error { +func (re *runExtractor) extractLog(cr flux.ColReader) error { entries := make(map[platform.ID][]string) for i := 0; i < cr.Len(); i++ { var runID platform.ID