From aa9c49e9f548ed5c9d2b91996b8ea654ae26b603 Mon Sep 17 00:00:00 2001 From: "Christopher M. Wolff" Date: Wed, 24 Aug 2022 11:59:16 -0700 Subject: [PATCH] build(flux): update flux to v0.180.1 (#23666) * chore: make tests robust to Flux formatter changes --- checks/service_external_test.go | 6 +++ etc/test-flux.sh | 31 ++++++++++++- go.mod | 2 +- go.sum | 4 +- http/check_test.go | 44 ++++++++++++++++++- notification/check/deadman_test.go | 3 +- notification/check/threshold_test.go | 3 +- notification/rule/http_test.go | 17 +++---- notification/rule/pagerduty_test.go | 3 +- .../rule/service/service_external_test.go | 5 ++- notification/rule/slack_test.go | 3 +- notification/rule/telegram_test.go | 2 +- query/fluxlang/service.go | 8 ++++ testing/util.go | 11 +++++ 14 files changed, 121 insertions(+), 21 deletions(-) diff --git a/checks/service_external_test.go b/checks/service_external_test.go index 29791bff2a..a6abc97845 100644 --- a/checks/service_external_test.go +++ b/checks/service_external_test.go @@ -18,6 +18,7 @@ import ( "github.com/influxdata/influxdb/v2/notification" "github.com/influxdata/influxdb/v2/notification/check" "github.com/influxdata/influxdb/v2/task/taskmodel" + itesting "github.com/influxdata/influxdb/v2/testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -173,6 +174,11 @@ var taskCmpOptions = cmp.Options{ }) return out }), + cmp.Transformer("FormatFlux", func(in taskmodel.Task) taskmodel.Task { + newTask := in + newTask.Flux = itesting.FormatFluxString(&testing.T{}, newTask.Flux) + return newTask + }), } // CheckFields will include the IDGenerator, and checks diff --git a/etc/test-flux.sh b/etc/test-flux.sh index dbd35bafa5..3e9a3bf790 100755 --- a/etc/test-flux.sh +++ b/etc/test-flux.sh @@ -76,7 +76,36 @@ remove_sort_selector remove_sort_filter_range remove_sort_aggregate_window remove_sort_join - +vec_conditional_time +vec_conditional_time_repeat +vec_conditional_int +vec_conditional_int_repeat +vec_conditional_float +vec_conditional_float_repeat +vec_conditional_uint +vec_conditional_string +vec_conditional_string_repeat +vec_const_with_const +vec_const_with_const_add_const +vec_const_add_member_const +vec_const_with_const_add_const_add_member +vec_const_with_const_add_member_add_const +vec_const_with_member_add_const_add_const +vec_const_kitchen_sink_column_types +vec_equality_time +vec_equality_time_repeat +vec_equality_int +vec_equality_int_repeat +vec_equality_float +vec_equality_float_repeat +vec_equality_uint +vec_equality_string +vec_equality_string_repeat +vec_equality_bool +vec_equality_casts +vec_with_float +vec_with_float_const + # Other skipped tests align_time buckets diff --git a/go.mod b/go.mod index 16743c49ca..e7ee68a39e 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/google/go-jsonnet v0.17.0 github.com/hashicorp/vault/api v1.0.2 github.com/influxdata/cron v0.0.0-20201006132531-4bb0a200dcbe - github.com/influxdata/flux v0.179.0 + github.com/influxdata/flux v0.180.1 github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69 github.com/influxdata/influx-cli/v2 v2.2.1-0.20220318222112-88ba3464cd07 github.com/influxdata/influxql v1.1.1-0.20211004132434-7e7d61973256 diff --git a/go.sum b/go.sum index 6ceb59b24c..932118e3c9 100644 --- a/go.sum +++ b/go.sum @@ -503,8 +503,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/cron v0.0.0-20201006132531-4bb0a200dcbe h1:7j4SdN/BvQwN6WoUq7mv0kg5U9NhnFBxPGMafYRKym0= github.com/influxdata/cron v0.0.0-20201006132531-4bb0a200dcbe/go.mod h1:XabtPPW2qsCg0tl+kjaPU+cFS+CjQXEXbT1VJvHT4og= -github.com/influxdata/flux v0.179.0 h1:Mk85dwwN+KcHAj3lT0nwITGY++x6q6aKrXx/M7789fo= -github.com/influxdata/flux v0.179.0/go.mod h1:FY4XAM6a/04xK/pyKyGKQJ3Ovo98h87I5cbT1758b0M= +github.com/influxdata/flux v0.180.1 h1:zM+U3LgV+qWuAeixGeSm0+AVYSlnZAT71H/JSwLAsSc= +github.com/influxdata/flux v0.180.1/go.mod h1:FY4XAM6a/04xK/pyKyGKQJ3Ovo98h87I5cbT1758b0M= github.com/influxdata/gosnowflake v1.6.9 h1:BhE39Mmh8bC+Rvd4QQsP2gHypfeYIH1wqW1AjGWxxrE= github.com/influxdata/gosnowflake v1.6.9/go.mod h1:9W/BvCXOKx2gJtQ+jdi1Vudev9t9/UDOEHnlJZ/y1nU= github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69 h1:WQsmW0fXO4ZE/lFGIE84G6rIV5SJN3P3sjIXAP1a8eU= diff --git a/http/check_test.go b/http/check_test.go index 5fcf92085e..8952a46c04 100644 --- a/http/check_test.go +++ b/http/check_test.go @@ -418,7 +418,31 @@ func TestService_handleGetCheckQuery(t *testing.T) { wants: wants{ statusCode: http.StatusOK, contentType: "application/json; charset=utf-8", - body: "{\"flux\":\"import \\\"influxdata/influxdb/monitor\\\"\\nimport \\\"influxdata/influxdb/v1\\\"\\n\\ndata =\\n from(bucket: \\\"foo\\\")\\n |\\u003e range(start: -1h)\\n |\\u003e filter(fn: (r) =\\u003e r._field == \\\"usage_idle\\\")\\n |\\u003e aggregateWindow(every: 1h, fn: mean, createEmpty: false)\\n\\noption task = {name: \\\"hello\\\", every: 1h}\\n\\ncheck = {_check_id: \\\"020f755c3c082000\\\", _check_name: \\\"hello\\\", _type: \\\"threshold\\\", tags: {aaa: \\\"vaaa\\\", bbb: \\\"vbbb\\\"}}\\nok = (r) =\\u003e r[\\\"usage_idle\\\"] \\u003e 10.0\\ninfo = (r) =\\u003e r[\\\"usage_idle\\\"] \\u003c 40.0\\nwarn = (r) =\\u003e r[\\\"usage_idle\\\"] \\u003c 40.0 and r[\\\"usage_idle\\\"] \\u003e 10.0\\ncrit = (r) =\\u003e r[\\\"usage_idle\\\"] \\u003c 40.0 and r[\\\"usage_idle\\\"] \\u003e 10.0\\nmessageFn = (r) =\\u003e \\\"whoa! {check.yeah}\\\"\\n\\ndata\\n |\\u003e v1[\\\"fieldsAsCols\\\"]()\\n |\\u003e monitor[\\\"check\\\"](\\n data: check,\\n messageFn: messageFn,\\n ok: ok,\\n info: info,\\n warn: warn,\\n crit: crit,\\n )\\n\"}\n", + body: "{\"flux\":" + formatFluxJson(t, `import "influxdata/influxdb/monitor" +import "influxdata/influxdb/v1" +data = + from(bucket: "foo") + |> range(start: -1h) + |> filter(fn: (r) => r._field == "usage_idle") + |> aggregateWindow(every: 1h, fn: mean, createEmpty: false) +option task = {name: "hello", every: 1h} +check = {_check_id: "020f755c3c082000", _check_name: "hello", _type: "threshold", tags: {aaa: "vaaa", bbb: "vbbb"}} +ok = (r) => r["usage_idle"] > 10.0 +info = (r) => r["usage_idle"] < 40.0 +warn = (r) => r["usage_idle"] < 40.0 and r["usage_idle"] > 10.0 +crit = (r) => r["usage_idle"] < 40.0 and r["usage_idle"] > 10.0 +messageFn = (r) => "whoa! {check.yeah}" +data + |> v1["fieldsAsCols"]() + |> monitor["check"]( + data: check, + messageFn: messageFn, + ok: ok, + info: info, + warn: warn, + crit: crit, + ) +`) + "}\n", }, }, } @@ -445,7 +469,9 @@ func TestService_handleGetCheckQuery(t *testing.T) { }). ExpectBody(func(body *bytes.Buffer) { if eq, diff, err := jsonEqual(body.String(), tt.wants.body); err != nil || tt.wants.body != "" && !eq { - fmt.Printf("%q\n", body.String()) + if err != nil { + t.Errorf("jsonEqual error: %v", err) + } t.Errorf("%q. handleGetChecks() = ***%v***", tt.name, diff) } }) @@ -453,6 +479,20 @@ func TestService_handleGetCheckQuery(t *testing.T) { } } +func formatFluxJson(t *testing.T, script string) string { + formatted := influxTesting.FormatFluxString(t, script) + + enc, err := json.Marshal(formatted) + if err != nil { + t.Fatalf("error marshalling flux: %v", err) + } + + var bb bytes.Buffer + json.HTMLEscape(&bb, enc) + std := bb.String() + return std +} + func TestService_handleGetCheck(t *testing.T) { type fields struct { CheckService influxdb.CheckService diff --git a/notification/check/deadman_test.go b/notification/check/deadman_test.go index 70ee357f58..dc352c879d 100644 --- a/notification/check/deadman_test.go +++ b/notification/check/deadman_test.go @@ -7,6 +7,7 @@ import ( "github.com/influxdata/influxdb/v2/notification" "github.com/influxdata/influxdb/v2/notification/check" "github.com/influxdata/influxdb/v2/query/fluxlang" + itesting "github.com/influxdata/influxdb/v2/testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -195,7 +196,7 @@ data t.Run(tt.name, func(t *testing.T) { s, err := tt.args.deadman.GenerateFlux(fluxlang.DefaultService) require.NoError(t, err) - assert.Equal(t, tt.wants.script, s) + assert.Equal(t, itesting.FormatFluxString(t, tt.wants.script), s) }) } diff --git a/notification/check/threshold_test.go b/notification/check/threshold_test.go index db7f5854aa..0b1ae5697f 100644 --- a/notification/check/threshold_test.go +++ b/notification/check/threshold_test.go @@ -7,6 +7,7 @@ import ( "github.com/influxdata/influxdb/v2/notification" "github.com/influxdata/influxdb/v2/notification/check" "github.com/influxdata/influxdb/v2/query/fluxlang" + itesting "github.com/influxdata/influxdb/v2/testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -357,7 +358,7 @@ data t.Run(tt.name, func(t *testing.T) { s, err := tt.args.threshold.GenerateFlux(fluxlang.DefaultService) require.NoError(t, err) - assert.Equal(t, tt.wants.script, s) + assert.Equal(t, itesting.FormatFluxString(t, tt.wants.script), s) }) } diff --git a/notification/rule/http_test.go b/notification/rule/http_test.go index 6ffc36e66d..f9d8802c25 100644 --- a/notification/rule/http_test.go +++ b/notification/rule/http_test.go @@ -9,12 +9,13 @@ import ( "github.com/influxdata/influxdb/v2/notification" "github.com/influxdata/influxdb/v2/notification/endpoint" "github.com/influxdata/influxdb/v2/notification/rule" + itesting "github.com/influxdata/influxdb/v2/testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestHTTP_GenerateFlux(t *testing.T) { - want := `import "influxdata/influxdb/monitor" + want := itesting.FormatFluxString(t, `import "influxdata/influxdb/monitor" import "http" import "json" import "experimental" @@ -45,7 +46,7 @@ all_statuses }, ), ) -` +`) s := &rule.HTTP{ Base: rule.Base{ @@ -83,7 +84,7 @@ all_statuses } func TestHTTP_GenerateFlux_basicAuth(t *testing.T) { - want := `import "influxdata/influxdb/monitor" + want := itesting.FormatFluxString(t, `import "influxdata/influxdb/monitor" import "http" import "json" import "experimental" @@ -122,7 +123,7 @@ all_statuses }, ), ) -` +`) s := &rule.HTTP{ Base: rule.Base{ ID: 1, @@ -166,7 +167,7 @@ all_statuses } func TestHTTP_GenerateFlux_bearer(t *testing.T) { - want := `import "influxdata/influxdb/monitor" + want := itesting.FormatFluxString(t, `import "influxdata/influxdb/monitor" import "http" import "json" import "experimental" @@ -201,7 +202,7 @@ all_statuses }, ), ) -` +`) s := &rule.HTTP{ Base: rule.Base{ @@ -243,7 +244,7 @@ all_statuses } func TestHTTP_GenerateFlux_bearer_every_second(t *testing.T) { - want := `import "influxdata/influxdb/monitor" + want := itesting.FormatFluxString(t, `import "influxdata/influxdb/monitor" import "http" import "json" import "experimental" @@ -278,7 +279,7 @@ all_statuses }, ), ) -` +`) s := &rule.HTTP{ Base: rule.Base{ diff --git a/notification/rule/pagerduty_test.go b/notification/rule/pagerduty_test.go index f89adfff8a..c31652acae 100644 --- a/notification/rule/pagerduty_test.go +++ b/notification/rule/pagerduty_test.go @@ -8,6 +8,7 @@ import ( "github.com/influxdata/influxdb/v2/notification" "github.com/influxdata/influxdb/v2/notification/endpoint" "github.com/influxdata/influxdb/v2/notification/rule" + itesting "github.com/influxdata/influxdb/v2/testing" ) func TestPagerDuty_GenerateFlux(t *testing.T) { @@ -284,7 +285,7 @@ all_statuses panic(err) } - if got, want := script, tt.script; got != want { + if got, want := script, itesting.FormatFluxString(t, tt.script); got != want { t.Errorf("\n\nStrings do not match:\n\n%s", diff.LineDiff(got, want)) } diff --git a/notification/rule/service/service_external_test.go b/notification/rule/service/service_external_test.go index 8e82c8ef0e..df090f6eea 100644 --- a/notification/rule/service/service_external_test.go +++ b/notification/rule/service/service_external_test.go @@ -20,6 +20,7 @@ import ( "github.com/influxdata/influxdb/v2/notification/rule" "github.com/influxdata/influxdb/v2/pkg/pointer" "github.com/influxdata/influxdb/v2/task/taskmodel" + itesting "github.com/influxdata/influxdb/v2/testing" ) const ( @@ -295,7 +296,7 @@ func CreateNotificationRule( OwnerID: MustIDBase16("020f755c3c082005"), Name: "name2", Status: "active", - Flux: `import "influxdata/influxdb/monitor" + Flux: itesting.FormatFluxString(t, `import "influxdata/influxdb/monitor" import "slack" import "influxdata/influxdb/secrets" import "experimental" @@ -333,7 +334,7 @@ all_statuses }), ), ) -`, +`), Every: "1h", }, }, diff --git a/notification/rule/slack_test.go b/notification/rule/slack_test.go index 56f19ee641..e4e0dd3372 100644 --- a/notification/rule/slack_test.go +++ b/notification/rule/slack_test.go @@ -10,6 +10,7 @@ import ( "github.com/influxdata/influxdb/v2/notification" "github.com/influxdata/influxdb/v2/notification/endpoint" "github.com/influxdata/influxdb/v2/notification/rule" + itesting "github.com/influxdata/influxdb/v2/testing" ) func mustDuration(d string) *notification.Duration { @@ -393,7 +394,7 @@ all_statuses t.Fatal(err) } - if f != tt.want { + if f != itesting.FormatFluxString(t, tt.want) { t.Errorf("scripts did not match. want:\n%v\n\ngot:\n%v", tt.want, f) } }) diff --git a/notification/rule/telegram_test.go b/notification/rule/telegram_test.go index 4ef7cc4414..86552dc55f 100644 --- a/notification/rule/telegram_test.go +++ b/notification/rule/telegram_test.go @@ -224,7 +224,7 @@ all_statuses return } - if got, want := script, tt.script; got != want { + if got, want := script, influxTesting.FormatFluxString(t, tt.script); got != want { t.Errorf("\n\nStrings do not match:\n\n%s", diff.LineDiff(got, want)) } }) diff --git a/query/fluxlang/service.go b/query/fluxlang/service.go index 44d7ca53df..d50f16d296 100644 --- a/query/fluxlang/service.go +++ b/query/fluxlang/service.go @@ -5,6 +5,7 @@ import ( "context" "github.com/influxdata/flux/ast" + "github.com/influxdata/flux/ast/astutil" "github.com/influxdata/flux/complete" "github.com/influxdata/flux/interpreter" "github.com/influxdata/flux/parser" @@ -26,6 +27,9 @@ type FluxLanguageService interface { // but it may be null if parsing didn't even occur. Parse(source string) (*ast.Package, error) + // Format will produce a string for the given *ast.File. + Format(f *ast.File) (string, error) + // EvalAST will evaluate and run an AST. EvalAST(ctx context.Context, astPkg *ast.Package) ([]interpreter.SideEffect, values.Scope, error) @@ -46,6 +50,10 @@ func (d defaultService) Parse(source string) (pkg *ast.Package, err error) { return pkg, err } +func (d defaultService) Format(f *ast.File) (string, error) { + return astutil.Format(f) +} + func (d defaultService) EvalAST(ctx context.Context, astPkg *ast.Package) ([]interpreter.SideEffect, values.Scope, error) { return runtime.EvalAST(ctx, astPkg) } diff --git a/testing/util.go b/testing/util.go index 47147bfdc3..b2ed351a9e 100644 --- a/testing/util.go +++ b/testing/util.go @@ -13,6 +13,7 @@ import ( "github.com/influxdata/influxdb/v2/kit/platform/errors" "github.com/influxdata/influxdb/v2/kv" "github.com/influxdata/influxdb/v2/kv/migration/all" + "github.com/influxdata/influxdb/v2/query/fluxlang" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" @@ -155,3 +156,13 @@ func influxErrsEqual(t *testing.T, expected *errors.Error, actual error) { assert.Equal(t, expected.Code, iErr.Code) assert.Truef(t, strings.HasPrefix(iErr.Error(), expected.Error()), "expected: %s got err: %s", expected.Error(), actual.Error()) } + +func FormatFluxString(t *testing.T, script string) string { + svc := fluxlang.DefaultService + + astPkg, err := svc.Parse(script) + require.NoError(t, err) + formatted, err := svc.Format(astPkg.Files[0]) + require.NoError(t, err) + return formatted +}