Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(influx): Add Flux support, enabled by specifying -type=flux

pull/10413/head
Stuart Carnie 2018-10-24 14:06:54 -07:00
parent 9d1a8c97d8
commit c5ec3a3244
7 changed files with 406 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
Gopkg.lock generated
View File

@ -148,6 +148,14 @@
revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
version = "v1.3.1"
[[projects]]
digest = "1:1659cb76cbd08a29826688d006e7a3d9279d6ca8a12155acb7b20164958987d3"
name = "github.com/c-bata/go-prompt"
packages = ["."]
pruneopts = "UT"
revision = "e99fbc797b795e0a7a94affc8d44f6a0350d85f0"
version = "v0.2.1"
[[projects]]
digest = "1:bf525707b4455ed126b3b091939a3693f608385c73d8dfdeb748e510bcda1338"
name = "github.com/caarlos0/ctrlc"
@ -388,7 +396,7 @@
[[projects]]
branch = "master"
digest = "1:cc977ce9615b1e9e23fe61d1fb36845a971199193c174968fdb017ec58913b79"
digest = "1:6be98dfd22bde6335f3a675f01c2498597f02d7940f4789ccf6b1ab0ad36c8d5"
name = "github.com/influxdata/flux"
packages = [
".",
@ -405,6 +413,7 @@
"options",
"parser",
"plan",
"repl",
"semantic",
"values",
]
@ -575,6 +584,14 @@
revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
version = "v0.0.2"
[[projects]]
branch = "master"
digest = "1:325f68c3bd3044dcf51d46e802136cd239817e1fb130b5f3466a4cec4dd427d3"
name = "github.com/mattn/go-tty"
packages = ["."]
pruneopts = "UT"
revision = "13ff1204f104d52c3f7645ec027ecbcf9026429e"
[[projects]]
branch = "master"
digest = "1:36aebe90a13cf9128280ac834399b8bebf83685283c78df279d61c46bb2a8d83"
@ -666,6 +683,14 @@
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
branch = "master"
digest = "1:4f43b3c1b7e44980a5f3c593f8bf0e18844dc44f451a071c93e77e28cf990db6"
name = "github.com/pkg/term"
packages = ["termios"]
pruneopts = "UT"
revision = "bffc007b7fd5a70e20e28f5b7649bb84671ef436"
[[projects]]
digest = "1:70f78dea42b8c0ff38ecf5487eaa79006fa2193fc804fc7c1d7222745d9e2522"
name = "github.com/prometheus/client_golang"
@ -972,6 +997,7 @@
"github.com/influxdata/flux/lang",
"github.com/influxdata/flux/options",
"github.com/influxdata/flux/plan",
"github.com/influxdata/flux/repl",
"github.com/influxdata/flux/values",
"github.com/influxdata/influxql",
"github.com/influxdata/platform/models",
@ -990,6 +1016,7 @@
"github.com/paulbellamy/ratecounter",
"github.com/peterh/liner",
"github.com/pkg/errors",
"github.com/prometheus/client_golang/prometheus",
"github.com/prometheus/client_golang/prometheus/promhttp",
"github.com/retailnext/hllpp",
"github.com/tinylib/msgp/msgp",

63
cmd/influx/cli/cli.go
View File

@ -143,8 +143,7 @@ func (c *CommandLine) Run() error {
if c.Execute != "" {
switch c.Type {
case QueryLanguageFlux:
// execute Flux query
fmt.Println("Execute Flux query")
return c.ExecuteFluxQuery(c.Execute)
default:
// Make the non-interactive mode send everything through the CLI's parser
// the same way the interactive mode works
@ -187,9 +186,7 @@ func (c *CommandLine) Run() error {
switch c.Type {
case QueryLanguageFlux:
// execute Flux query
fmt.Println("Read STDIN and execute Flux query")
return nil
return c.ExecuteFluxQuery(string(cmd))
default:
return c.ExecuteQuery(string(cmd))
}
@ -200,17 +197,6 @@ func (c *CommandLine) Run() error {
signal.Notify(c.osSignals, syscall.SIGINT, syscall.SIGTERM)
}
if c.Type == QueryLanguageFlux {
// execute Flux repl
fmt.Println("Execute Flux REPL")
return nil
}
c.Line = liner.NewLiner()
defer c.Line.Close()
c.Line.SetMultiLineMode(true)
if len(c.ServerVersion) == 0 {
fmt.Printf("WARN: Connected to %s, but found no server version.\n", c.Client.Addr())
fmt.Printf("Are you sure an InfluxDB server is listening at the given address?\n")
@ -220,6 +206,20 @@ func (c *CommandLine) Run() error {
c.Version()
if c.Type == QueryLanguageFlux {
repl, err := getFluxREPL(c.Host, c.Port, c.Ssl)
if err != nil {
return err
}
repl.Run()
os.Exit(0)
}
c.Line = liner.NewLiner()
defer c.Line.Close()
c.Line.SetMultiLineMode(true)
// Only load/write history if HOME environment variable is set.
var historyDir string
if runtime.GOOS == "windows" {
@ -1161,6 +1161,12 @@ func (c *CommandLine) gopher() {
// Version prints the CLI version.
func (c *CommandLine) Version() {
fmt.Println("InfluxDB shell version:", c.ClientVersion)
switch c.Type {
case QueryLanguageFlux:
fmt.Println("Enter a Flux query")
default:
fmt.Println("Enter an InfluxQL query")
}
}
func (c *CommandLine) exit() {
@ -1171,6 +1177,31 @@ func (c *CommandLine) exit() {
c.Line = nil
}
func (c *CommandLine) ExecuteFluxQuery(query string) error {
ctx := context.Background()
if !c.IgnoreSignals {
done := make(chan struct{})
defer close(done)
var cancel func()
ctx, cancel = context.WithCancel(ctx)
go func() {
select {
case <-done:
case <-c.osSignals:
cancel()
}
}()
}
repl, err := getFluxREPL(c.Host, c.Port, c.Ssl)
if err != nil {
return err
}
return repl.Input(query)
}
type QueryLanguage uint8
const (

39
cmd/influx/cli/flux.go Normal file
View File

@ -0,0 +1,39 @@
package cli
import (
"context"
"github.com/influxdata/flux"
"github.com/influxdata/flux/csv"
"github.com/influxdata/flux/repl"
_ "github.com/influxdata/influxdb/flux/builtin"
"github.com/influxdata/influxdb/flux/client"
)
// QueryService represents a type capable of performing queries.
type fluxClient interface {
// Query submits a query for execution returning a results iterator.
// Cancel must be called on any returned results to free resources.
Query(ctx context.Context, req *client.ProxyRequest) (flux.ResultIterator, error)
}
// replQuerier implements the repl.Querier interface while consuming a fluxClient
type replQuerier struct {
client fluxClient
}
func (q *replQuerier) Query(ctx context.Context, compiler flux.Compiler) (flux.ResultIterator, error) {
req := &client.ProxyRequest{
Compiler: compiler,
Dialect: csv.DefaultDialect(),
}
return q.client.Query(ctx, req)
}
func getFluxREPL(host string, port int, ssl bool) (*repl.REPL, error) {
c, err := client.NewHTTP(host, port, ssl)
if err != nil {
return nil, err
}
return repl.New(&replQuerier{client: c}), nil
}

150
flux/client/http.go Normal file
View File

@ -0,0 +1,150 @@
package client
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"net/http"
"net/url"
"strconv"
"github.com/influxdata/flux"
"github.com/influxdata/flux/csv"
"github.com/influxdata/flux/lang"
iclient "github.com/influxdata/influxdb/client"
"github.com/pkg/errors"
)
const (
fluxPath = "/api/v2/query"
)
// Shared transports for all clients to prevent leaking connections
var (
skipVerifyTransport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
defaultTransport = &http.Transport{}
)
// HTTP implements a Flux query client that makes requests to the /api/v2/query
// API endpoint.
type HTTP struct {
Addr string
InsecureSkipVerify bool
url *url.URL
}
// NewHTTP creates a HTTP client
func NewHTTP(host string, port int, ssl bool) (*HTTP, error) {
addr := net.JoinHostPort(host, strconv.Itoa(port))
u, e := iclient.ParseConnectionString(addr, ssl)
if e != nil {
return nil, e
}
u.Path = fluxPath
return &HTTP{url: &u}, nil
}
// Query runs a flux query against a influx server and decodes the result
func (s *HTTP) Query(ctx context.Context, r *ProxyRequest) (flux.ResultIterator, error) {
qreq, err := QueryRequestFromProxyRequest(r)
if err != nil {
return nil, err
}
var body bytes.Buffer
if err := json.NewEncoder(&body).Encode(qreq); err != nil {
return nil, err
}
hreq, err := http.NewRequest("POST", s.url.String(), &body)
if err != nil {
return nil, err
}
hreq.Header.Set("Content-Type", "application/json")
hreq.Header.Set("Accept", "text/csv")
hreq = hreq.WithContext(ctx)
hc := newClient(s.url.Scheme, s.InsecureSkipVerify)
resp, err := hc.Do(hreq)
if err != nil {
return nil, err
}
if err := checkError(resp); err != nil {
return nil, err
}
decoder := csv.NewMultiResultDecoder(csv.ResultDecoderConfig{})
return decoder.Decode(resp.Body)
}
func newClient(scheme string, insecure bool) *http.Client {
hc := &http.Client{
Transport: defaultTransport,
}
if scheme == "https" && insecure {
hc.Transport = skipVerifyTransport
}
return hc
}
// CheckError reads the http.Response and returns an error if one exists.
// It will automatically recognize the errors returned by Influx services
// and decode the error into an internal error type. If the error cannot
// be determined in that way, it will create a generic error message.
//
// If there is no error, then this returns nil.
func checkError(resp *http.Response) error {
switch resp.StatusCode / 100 {
case 4, 5:
// We will attempt to parse this error outside of this block.
case 2:
return nil
default:
// TODO(jsternberg): Figure out what to do here?
//return kerrors.InternalErrorf("unexpected status code: %d %s", resp.StatusCode, resp.Status)
}
// There is no influx error so we need to report that we have some kind
// of error from somewhere.
// TODO(jsternberg): Try to make this more advance by reading the response
// and either decoding a possible json message or just reading the text itself.
// This might be good enough though.
msg := "unknown server error"
if resp.StatusCode/100 == 4 {
msg = "client error"
}
return errors.Wrap(errors.New(resp.Status), msg)
}
func QueryRequestFromProxyRequest(req *ProxyRequest) (*QueryRequest, error) {
qr := new(QueryRequest)
switch c := req.Compiler.(type) {
case lang.FluxCompiler:
qr.Type = "flux"
qr.Query = c.Query
case lang.SpecCompiler:
qr.Type = "flux"
qr.Spec = c.Spec
default:
return nil, fmt.Errorf("unsupported compiler %T", c)
}
switch d := req.Dialect.(type) {
case *csv.Dialect:
var header = !d.ResultEncoderConfig.NoHeader
qr.Dialect.Header = &header
qr.Dialect.Delimiter = string(d.ResultEncoderConfig.Delimiter)
qr.Dialect.CommentPrefix = "#"
qr.Dialect.DateTimeFormat = "RFC3339"
qr.Dialect.Annotations = d.ResultEncoderConfig.Annotations
default:
return nil, fmt.Errorf("unsupported dialect %T", d)
}
return qr, nil
}

135
flux/client/request.go Normal file
View File

@ -0,0 +1,135 @@
package client
import (
"context"
"fmt"
"unicode/utf8"
"github.com/influxdata/flux"
"github.com/influxdata/flux/csv"
"github.com/influxdata/flux/lang"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
)
type Controller interface {
Query(ctx context.Context, compiler flux.Compiler) (flux.Query, error)
PrometheusCollectors() []prometheus.Collector
}
// QueryRequest is a flux query request.
type QueryRequest struct {
Spec *flux.Spec `json:"spec,omitempty"`
Query string `json:"query"`
Type string `json:"type"`
Dialect QueryDialect `json:"dialect"`
}
// QueryDialect is the formatting options for the query response.
type QueryDialect struct {
Header *bool `json:"header"`
Delimiter string `json:"delimiter"`
CommentPrefix string `json:"commentPrefix"`
DateTimeFormat string `json:"dateTimeFormat"`
Annotations []string `json:"annotations"`
}
// WithDefaults adds default values to the request.
func (r QueryRequest) WithDefaults() QueryRequest {
if r.Type == "" {
r.Type = "flux"
}
if r.Dialect.Delimiter == "" {
r.Dialect.Delimiter = ","
}
if r.Dialect.DateTimeFormat == "" {
r.Dialect.DateTimeFormat = "RFC3339"
}
if r.Dialect.Header == nil {
header := true
r.Dialect.Header = &header
}
return r
}
// Validate checks the query request and returns an error if the request is invalid.
func (r QueryRequest) Validate() error {
if r.Query == "" && r.Spec == nil {
return errors.New(`request body requires either spec or query`)
}
if r.Type != "flux" {
return fmt.Errorf(`unknown query type: %s`, r.Type)
}
if len(r.Dialect.CommentPrefix) > 1 {
return fmt.Errorf("invalid dialect comment prefix: must be length 0 or 1")
}
if len(r.Dialect.Delimiter) != 1 {
return fmt.Errorf("invalid dialect delimeter: must be length 1")
}
rn, size := utf8.DecodeRuneInString(r.Dialect.Delimiter)
if rn == utf8.RuneError && size == 1 {
return fmt.Errorf("invalid dialect delimeter character")
}
for _, a := range r.Dialect.Annotations {
switch a {
case "group", "datatype", "default":
default:
return fmt.Errorf(`unknown dialect annotation type: %s`, a)
}
}
switch r.Dialect.DateTimeFormat {
case "RFC3339", "RFC3339Nano":
default:
return fmt.Errorf(`unknown dialect date time format: %s`, r.Dialect.DateTimeFormat)
}
return nil
}
// ProxyRequest specifies a query request and the dialect for the results.
type ProxyRequest struct {
// Compiler converts the query to a specification to run against the data.
Compiler flux.Compiler
// Dialect is the result encoder
Dialect flux.Dialect
}
// ProxyRequest returns a request to proxy from the flux.
func (r QueryRequest) ProxyRequest() *ProxyRequest {
// Query is preferred over spec
var compiler flux.Compiler
if r.Query != "" {
compiler = lang.FluxCompiler{
Query: r.Query,
}
} else if r.Spec != nil {
compiler = lang.SpecCompiler{
Spec: r.Spec,
}
}
delimiter, _ := utf8.DecodeRuneInString(r.Dialect.Delimiter)
noHeader := false
if r.Dialect.Header != nil {
noHeader = !*r.Dialect.Header
}
cfg := csv.DefaultEncoderConfig()
cfg.NoHeader = noHeader
cfg.Delimiter = delimiter
return &ProxyRequest{
Compiler: compiler,
Dialect: csv.Dialect{
ResultEncoderConfig: cfg,
},
}
}

127
services/httpd/flux.go
View File

@ -3,16 +3,12 @@ package httpd
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"mime"
"net/http"
"unicode/utf8"
"github.com/influxdata/flux"
"github.com/influxdata/flux/csv"
"github.com/influxdata/flux/lang"
"github.com/pkg/errors"
"github.com/influxdata/influxdb/flux/client"
"github.com/prometheus/client_golang/prometheus"
)
@ -21,136 +17,19 @@ type Controller interface {
PrometheusCollectors() []prometheus.Collector
}
// QueryRequest is a flux query request.
type QueryRequest struct {
Spec *flux.Spec `json:"spec,omitempty"`
Query string `json:"query"`
Type string `json:"type"`
Dialect QueryDialect `json:"dialect"`
}
// QueryDialect is the formatting options for the query response.
type QueryDialect struct {
Header *bool `json:"header"`
Delimiter string `json:"delimiter"`
CommentPrefix string `json:"commentPrefix"`
DateTimeFormat string `json:"dateTimeFormat"`
Annotations []string `json:"annotations"`
}
// WithDefaults adds default values to the request.
func (r QueryRequest) WithDefaults() QueryRequest {
if r.Type == "" {
r.Type = "flux"
}
if r.Dialect.Delimiter == "" {
r.Dialect.Delimiter = ","
}
if r.Dialect.DateTimeFormat == "" {
r.Dialect.DateTimeFormat = "RFC3339"
}
if r.Dialect.Header == nil {
header := true
r.Dialect.Header = &header
}
return r
}
// Validate checks the query request and returns an error if the request is invalid.
func (r QueryRequest) Validate() error {
if r.Query == "" && r.Spec == nil {
return errors.New(`request body requires either spec or query`)
}
if r.Type != "flux" {
return fmt.Errorf(`unknown query type: %s`, r.Type)
}
if len(r.Dialect.CommentPrefix) > 1 {
return fmt.Errorf("invalid dialect comment prefix: must be length 0 or 1")
}
if len(r.Dialect.Delimiter) != 1 {
return fmt.Errorf("invalid dialect delimeter: must be length 1")
}
rn, size := utf8.DecodeRuneInString(r.Dialect.Delimiter)
if rn == utf8.RuneError && size == 1 {
return fmt.Errorf("invalid dialect delimeter character")
}
for _, a := range r.Dialect.Annotations {
switch a {
case "group", "datatype", "default":
default:
return fmt.Errorf(`unknown dialect annotation type: %s`, a)
}
}
switch r.Dialect.DateTimeFormat {
case "RFC3339", "RFC3339Nano":
default:
return fmt.Errorf(`unknown dialect date time format: %s`, r.Dialect.DateTimeFormat)
}
return nil
}
// ProxyRequest specifies a query request and the dialect for the results.
type ProxyRequest struct {
// Compiler converts the query to a specification to run against the data.
Compiler flux.Compiler
// Dialect is the result encoder
Dialect flux.Dialect
}
// ProxyRequest returns a request to proxy from the flux.
func (r QueryRequest) ProxyRequest() *ProxyRequest {
// Query is preferred over spec
var compiler flux.Compiler
if r.Query != "" {
compiler = lang.FluxCompiler{
Query: r.Query,
}
} else if r.Spec != nil {
compiler = lang.SpecCompiler{
Spec: r.Spec,
}
}
delimiter, _ := utf8.DecodeRuneInString(r.Dialect.Delimiter)
noHeader := false
if r.Dialect.Header != nil {
noHeader = !*r.Dialect.Header
}
cfg := csv.DefaultEncoderConfig()
cfg.NoHeader = noHeader
cfg.Delimiter = delimiter
return &ProxyRequest{
Compiler: compiler,
Dialect: csv.Dialect{
ResultEncoderConfig: cfg,
},
}
}
// httpDialect is an encoding dialect that can write metadata to HTTP headers
type httpDialect interface {
SetHeaders(w http.ResponseWriter)
}
func decodeQueryRequest(r *http.Request) (*QueryRequest, error) {
func decodeQueryRequest(r *http.Request) (*client.QueryRequest, error) {
ct := r.Header.Get("Content-Type")
mt, _, err := mime.ParseMediaType(ct)
if err != nil {
return nil, err
}
var req QueryRequest
var req client.QueryRequest
switch mt {
case "application/vnd.flux":
if d, err := ioutil.ReadAll(r.Body); err != nil {

7
services/httpd/handler_test.go
View File

@ -26,6 +26,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/influxdata/flux"
"github.com/influxdata/flux/lang"
"github.com/influxdata/influxdb/flux/client"
"github.com/influxdata/influxdb/internal"
"github.com/influxdata/influxdb/logger"
"github.com/influxdata/influxdb/models"
@ -837,7 +838,7 @@ func TestHandler_Flux_QueryJSON(t *testing.T) {
return internal.NewFluxQueryMock(), nil
}
q := httpd.QueryRequest{Query: qry}
q := client.QueryRequest{Query: qry}
var body bytes.Buffer
if err := json.NewEncoder(&body).Encode(q); err != nil {
t.Fatalf("unexpected JSON encoding error: %q", err.Error())
@ -868,7 +869,7 @@ func TestHandler_Flux_SpecJSON(t *testing.T) {
return internal.NewFluxQueryMock(), nil
}
q := httpd.QueryRequest{Spec: &flux.Spec{}}
q := client.QueryRequest{Spec: &flux.Spec{}}
var body bytes.Buffer
if err := json.NewEncoder(&body).Encode(q); err != nil {
t.Fatalf("unexpected JSON encoding error: %q", err.Error())
@ -923,7 +924,7 @@ func TestHandler_Flux(t *testing.T) {
queryBytes := func(qs string) io.Reader {
var b bytes.Buffer
q := &httpd.QueryRequest{Query: qs}
q := &client.QueryRequest{Query: qs}
if err := json.NewEncoder(&b).Encode(q); err != nil {
t.Fatalf("unexpected JSON encoding error: %q", err.Error())
}