From cbeff50523dac925dfa424709e661b6c2e9e3ceb Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Mon, 10 Aug 2020 13:29:52 -0600 Subject: [PATCH 01/13] added flux 0.78.0 changelog --- content/v2.0/reference/release-notes/flux.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/content/v2.0/reference/release-notes/flux.md b/content/v2.0/reference/release-notes/flux.md index 84f92d94d..777415d37 100644 --- a/content/v2.0/reference/release-notes/flux.md +++ b/content/v2.0/reference/release-notes/flux.md @@ -16,6 +16,19 @@ Though newer versions of Flux may be available, they will not be included with InfluxDB until the next InfluxDB v2.0 release._ {{% /note %}} +## v0.78.0 [2020-08-10] + +### Features +- Add functions to convert semantic monotype to AST type. +- Add BigQuery support. +- Rust flatbuffer serialization for `MonoType` and `TypeExpression`. +- Extend with Geo package functions and add unit support. + +### Bug fixes +- String interpolation in arrays. + +--- + ## v0.77.1 [2020-08-03] ### Bug fixes From 71f5651516fec0102c19eede6265589780fe54fd Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 09:16:06 -0600 Subject: [PATCH 02/13] added new GIS functions and unit support to geo package --- .../flux/stdlib/experimental/geo/_index.md | 50 +++++++++++ .../stdlib/experimental/geo/filterrows.md | 31 ++++--- .../stdlib/experimental/geo/s2celllatlon.md | 36 ++++++++ .../flux/stdlib/experimental/geo/shapedata.md | 9 +- .../stdlib/experimental/geo/st_contains.md | 86 +++++++++++++++++++ .../stdlib/experimental/geo/st_distance.md | 85 ++++++++++++++++++ .../stdlib/experimental/geo/st_dwithin.md | 75 ++++++++++++++++ .../stdlib/experimental/geo/st_intersects.md | 66 ++++++++++++++ .../flux/stdlib/experimental/geo/st_length.md | 62 +++++++++++++ .../stdlib/experimental/geo/st_linestring.md | 69 +++++++++++++++ .../flux/stdlib/experimental/geo/torows.md | 18 +--- content/v2.0/reference/release-notes/flux.md | 3 +- 12 files changed, 555 insertions(+), 35 deletions(-) create mode 100644 content/v2.0/reference/flux/stdlib/experimental/geo/s2celllatlon.md create mode 100644 content/v2.0/reference/flux/stdlib/experimental/geo/st_contains.md create mode 100644 content/v2.0/reference/flux/stdlib/experimental/geo/st_distance.md create mode 100644 content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md create mode 100644 content/v2.0/reference/flux/stdlib/experimental/geo/st_intersects.md create mode 100644 content/v2.0/reference/flux/stdlib/experimental/geo/st_length.md create mode 100644 content/v2.0/reference/flux/stdlib/experimental/geo/st_linestring.md diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md b/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md index e1d1986e0..2a96d422e 100644 --- a/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md @@ -80,6 +80,7 @@ Define geographic regions using the following shapes: - [box](#box) - [circle](#circle) +- [point](#point) - [polygon](#polygon) ### box @@ -116,6 +117,20 @@ Define a circular region by specifying an object containing the following proper } ``` +### point +Define a point region by specifying and object containing the following properties: + +- **lat**: latitude in decimal degrees (WGS 84) _(Float)_ +- **lon**: longitude in decimal degrees (WGS 84) _(Float)_ + +##### Example point region +```js +{ + lat: 40.671659, + lon: -73.936631 +} +``` + ### polygon Define a custom polygon region using an object containing the following properties: @@ -136,3 +151,38 @@ Define a custom polygon region using an object containing the following properti ] } ``` + +## GIS geometry definitions +Many functions in the Geo package manipulate data based on geographic information system (GIS) data. +Define GIS geometry using the following: + +- Any [region type](#region-definitions) _(typically [point](#point))_ +- [linestring](#linestring) + +### linestring +Define a geographic linestring path using an object containing the following properties: + +- **linestring**: string containing comma-separatedlongitude and latitude + coordinate pairs (`lon lat,`): + +```js +{ + linestring: "39.7515 14.01433, 38.3527 13.9228, 36.9978 15.08433" +} +``` + +## Distance units +The Geo package supports the following units of measurement for distance: + +- `m` - meters +- `km` - kilometers _(default)_ +- `mile` - miles + +### Define distance units +Use the Geo package `units` option to define custom units of measurement: + +```js +import "experimental/geo" + +option geo.units = {distance: "mile"} +``` diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/filterrows.md b/content/v2.0/reference/flux/stdlib/experimental/geo/filterrows.md index 11c59240a..e7da63ad3 100644 --- a/content/v2.0/reference/flux/stdlib/experimental/geo/filterrows.md +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/filterrows.md @@ -196,19 +196,30 @@ filterRows = ( maxSize=-1, level=-1, s2cellIDLevel=-1, - correlationKey=["_time"], strict=true ) => { + _columns = + |> columns(column: "_value") + |> tableFind(fn: (key) => true ) + |> getColumn(column: "_value") _rows = - tables - |> gridFilter( - region, - minSize: minSize, - maxSize: maxSize, - level: level, - s2cellIDLevel: s2cellIDLevel - ) - |> toRows(correlationKey) + if contains(value: "lat", set: _columns) then + tables + |> gridFilter( + region: region, + minSize: minSize, + maxSize: maxSize, + level: level, + s2cellIDLevel: s2cellIDLevel) + else + tables + |> gridFilter( + region: region, + minSize: minSize, + maxSize: maxSize, + level: level, + s2cellIDLevel: s2cellIDLevel) + |> toRows() _result = if strict then _rows diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/s2celllatlon.md b/content/v2.0/reference/flux/stdlib/experimental/geo/s2celllatlon.md new file mode 100644 index 000000000..0593a5bfb --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/s2celllatlon.md @@ -0,0 +1,36 @@ +--- +title: geo.s2CellLatLon() function +description: > + The `geo.s2CellLatLon()` function returns the latitude and longitude of the + center of an S2 cell. +menu: + v2_0_ref: + name: geo.s2CellLatLon + parent: Geo +weight: 401 +v2.0/tags: [functions, geo] +related: + - /v2.0/query-data/flux/geo/ +--- + +The `geo.s2CellLatLon()` function returns the latitude and longitude of the +center of an S2 cell. + +_**Function type:** Transformation_ + +```js +import "experimental/geo" + +geo.s2CellLatLon( + token: "89c284" +) + +// Returns {lat: 40.812535546624574, lon: -73.55941282728273} +``` + +## Parameters + +### token +S2 cell ID token. + +_**Data type:** String_ diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/shapedata.md b/content/v2.0/reference/flux/stdlib/experimental/geo/shapedata.md index 4ab03fcf2..b519bb638 100644 --- a/content/v2.0/reference/flux/stdlib/experimental/geo/shapedata.md +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/shapedata.md @@ -32,8 +32,7 @@ import "experimental/geo" geo.shapeData( latField: "latitude", lonField: "longitude", - level: 10, - correlationKey: ["_time"] + level: 10 ) ``` @@ -57,12 +56,6 @@ when generating the S2 cell ID token. _**Data type:** Integer_ -### correlationKey -List of columns used to uniquely identify a row for output. -Default is `["_time"]`. - -_**Data type:** Array of strings_ - ## Examples ##### Shape data to meet the requirements of the Geo package diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/st_contains.md b/content/v2.0/reference/flux/stdlib/experimental/geo/st_contains.md new file mode 100644 index 000000000..ba8b37e58 --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/st_contains.md @@ -0,0 +1,86 @@ +--- +title: geo.ST_Contains() function +description: > + The `geo.ST_Contains()` function tests if the specified region contains the specified + GIS geometry and returns `true` or `false`. +menu: + v2_0_ref: + name: geo.ST_Contains + parent: Geo +weight: 401 +v2.0/tags: [functions, geo, GIS] +related: + - /v2.0/query-data/flux/geo/ +--- + +The `geo.ST_Contains()` function tests if the specified region contains the specified +geographic information system (GIS) geometry and returns `true` or `false`. + +_**Function type:** Test_ + +```js +import "experimental/geo" + +geo.ST_Contains( + region: {lat: 40.7, lon: -73.3, radius: 20.0}, + geometry: {lon: 39.7515, lat: 15.08433} +) + +// Returns false +``` + +## Parameters + +### region +The region to test. +Specify object properties for the shape. +_See [Region definitions](/v2.0/reference/flux/stdlib/experimental/geo/#region-definitions)._ + +_**Data type:** Object_ + +### geometry +The GIS geometry to test. +Can be either point or linestring geometry. +_See [GIS geometry definitions](/v2.0/reference/flux/stdlib/experimental/geo/#gis-geometry-definitions)._ + +_**Data type:** Object_ + +## Examples + +##### Test if geographic points are inside of a region +```js +import "experimental/geo" + +region = { + minLat: 40.51757813, + maxLat: 40.86914063, + minLon: -73.65234375, + maxLon: -72.94921875 +} + +data + |> geo.toRows() + |> map(fn: (r) => ({ + r with st_contains: geo.ST_Contains(region: region, geometry: {lat: r.lat, lon: r.lon}) + })) +``` + +##### Test if tracks are inside of a region +```js +import "experimental/geo" + +region = { + minLat: 40.51757813, + maxLat: 40.86914063, + minLon: -73.65234375, + maxLon: -72.94921875 +} + +data + |> geo.toRows() + |> geo.asTracks() + |> geo.ST_LineString() + |> map(fn: (r) => ({ + r with st_contains: geo.ST_Contains(region: region, geometry: {linestring: r.st_linestring}) + })) +``` diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/st_distance.md b/content/v2.0/reference/flux/stdlib/experimental/geo/st_distance.md new file mode 100644 index 000000000..dd51bb969 --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/st_distance.md @@ -0,0 +1,85 @@ +--- +title: geo.ST_Distance() function +description: > + The `geo.ST_Distance()` function returns the distance between the specified region + and specified GIS geometry. +menu: + v2_0_ref: + name: geo.ST_Distance + parent: Geo +weight: 401 +v2.0/tags: [functions, geo, GIS] +related: + - /v2.0/query-data/flux/geo/ +--- + +The `geo.ST_Distance()` function returns the distance between the specified region +and specified geographic information system (GIS) geometry. +Define distance units with the [`geo.units` option](/v2.0/reference/flux/stdlib/experimental/geo/#define-distance-units). + +_**Function type:** Transformation_ + +```js +import "experimental/geo" + +geo.ST_Distance( + region: {lat: 40.7, lon: -73.3, radius: 20.0}, + geometry: {lon: 39.7515, lat: 15.08433} +) + +// Returns 10734.184618677662 (km) +``` + +## Parameters + +### region +The region to test. +Specify object properties for the shape. +_See [Region definitions](/v2.0/reference/flux/stdlib/experimental/geo/#region-definitions)._ + +_**Data type:** Object_ + +### geometry +The GIS geometry to test. +Can be either point or linestring geometry. +_See [GIS geometry definitions](/v2.0/reference/flux/stdlib/experimental/geo/#gis-geometry-definitions)._ + +_**Data type:** Object_ + +## Examples + +##### Test if geographic points are inside of a region +```js +import "experimental/geo" + +region = { + minLat: 40.51757813, + maxLat: 40.86914063, + minLon: -73.65234375, + maxLon: -72.94921875 +} + +data + |> geo.toRows() + |> map(fn: (r) => ({ + r with st_contains: ST_Distance(region: region, geometry: {lat: r.lat, lon: r.lon}) + })) +``` + +##### Calculate the distance between geographic points and a region +```js +import "experimental/geo" + +region = { + minLat: 40.51757813, + maxLat: 40.86914063, + minLon: -73.65234375, + maxLon: -72.94921875 +} + +data + |> geo.toRows() + |> map(fn: (r) => ({ + r with st_distance: ST_Distance(region: region, geometry: {lat: r.lat, lon: r.lon}) + })) +``` diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md b/content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md new file mode 100644 index 000000000..4ee092ee4 --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md @@ -0,0 +1,75 @@ +--- +title: geo.ST_DWithin() function +description: > + The `geo.ST_DWithin()` function tests if the specified region is within a defined + distance from the specified geographic information system (GIS) geometry and + returns `true` or `false`. +menu: + v2_0_ref: + name: geo.ST_DWithin + parent: Geo +weight: 401 +v2.0/tags: [functions, geo, GIS] +related: + - /v2.0/query-data/flux/geo/ +--- + +The `geo.ST_DWithin()` function tests if the specified region is within a defined +distance from the specified geographic information system (GIS) geometry and +returns `true` or `false`. + +_**Function type:** Test_ + +```js +import "experimental/geo" + +geo.ST_DWithin( + region: {lat: 40.7, lon: -73.3, radius: 20.0}, + geometry: {lon: 39.7515, lat: 15.08433}, + distance: 1000.0 +) + +// Returns false +``` + +## Parameters + +### region +The region to test. +Specify object properties for the shape. +_See [Region definitions](/v2.0/reference/flux/stdlib/experimental/geo/#region-definitions)._ + +_**Data type:** Object_ + +### geometry +The GIS geometry to test. +Can be either point or linestring geometry. +_See [GIS geometry definitions](/v2.0/reference/flux/stdlib/experimental/geo/#gis-geometry-definitions)._ + +_**Data type:** Object_ + +### distance +Maximum allowed distance between the region and geometry. +_Define distance units with the [`geo.units` option](/v2.0/reference/flux/stdlib/experimental/geo/#define-distance-units)._ + +_**Data type:** Float_ + +## Examples + +##### Test if geographic points are within a distance from a region +```js +import "experimental/geo" + +region = { + minLat: 40.51757813, + maxLat: 40.86914063, + minLon: -73.65234375, + maxLon: -72.94921875 +} + +data + |> geo.toRows() + |> map(fn: (r) => ({ + r with st_within: geo.ST_DWithin(region: box, geometry: {lat: r.lat, lon: r.lon}, distance: 15.0) + })) +``` diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/st_intersects.md b/content/v2.0/reference/flux/stdlib/experimental/geo/st_intersects.md new file mode 100644 index 000000000..d0d3f2998 --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/st_intersects.md @@ -0,0 +1,66 @@ +--- +title: geo.ST_Intersects() function +description: > + The `geo.ST_Intersects()` function tests if the specified geographic information + system (GIS) geometry intersects with the specified region and returns `true` or `false`. +menu: + v2_0_ref: + name: geo.ST_Intersects + parent: Geo +weight: 401 +v2.0/tags: [functions, geo, GIS] +related: + - /v2.0/query-data/flux/geo/ +--- + +The `geo.ST_Intersects()` function tests if the specified geographic information +system (GIS) geometry intersects with the specified region and returns `true` or `false`. + +_**Function type:** Test_ + +```js +import "experimental/geo" + +geo.ST_Intersects( + region: {lat: 40.7, lon: -73.3, radius: 20.0}, + geometry: {linestring: "39.7515 14.01433, 38.3527 13.9228, 36.9978 15.08433"} +) + +// Returns false +``` + +## Parameters + +### region +The region to test. +Specify object properties for the shape. +_See [Region definitions](/v2.0/reference/flux/stdlib/experimental/geo/#region-definitions)._ + +_**Data type:** Object_ + +### geometry +The GIS geometry to test. +Can be either point or linestring geometry. +_See [GIS geometry definitions](/v2.0/reference/flux/stdlib/experimental/geo/#gis-geometry-definitions)._ + +_**Data type:** Object_ + +## Examples + +##### Test if geographic points intersect with a region +```js +import "experimental/geo" + +region = { + minLat: 40.51757813, + maxLat: 40.86914063, + minLon: -73.65234375, + maxLon: -72.94921875 +} + +data + |> geo.toRows() + |> map(fn: (r) => ({ + r with st_within: geo.ST_Intersects(region: box, geometry: {lat: r.lat, lon: r.lon}) + })) +``` diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/st_length.md b/content/v2.0/reference/flux/stdlib/experimental/geo/st_length.md new file mode 100644 index 000000000..33ce94a25 --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/st_length.md @@ -0,0 +1,62 @@ +--- +title: geo.ST_Length() function +description: > + The `geo.ST_Length()` function returns the [spherical length or distance](https://mathworld.wolfram.com/SphericalDistance.html) + of the specified geographic information system (GIS) geometry. +menu: + v2_0_ref: + name: geo.ST_Length + parent: Geo +weight: 401 +v2.0/tags: [functions, geo, GIS] +related: + - /v2.0/query-data/flux/geo/ +--- + +The `geo.ST_Length()` function returns the [spherical length or distance](https://mathworld.wolfram.com/SphericalDistance.html) +of the specified geographic information system (GIS) geometry. +Define distance units with the [`geo.units` option](/v2.0/reference/flux/stdlib/experimental/geo/#define-distance-units). + +_**Function type:** Transformation_ + +```js +import "experimental/geo" + +geo.ST_Length( + geometry: {linestring: "39.7515 14.01433, 38.3527 13.9228, 36.9978 15.08433"} +) + +// Returns 346.1023974652474 (km) +``` + +## Parameters + +### geometry +The GIS geometry to measure. +Can be either point or linestring geometry. +Points will always return `0.0`. +_See [GIS geometry definitions](/v2.0/reference/flux/stdlib/experimental/geo/#gis-geometry-definitions)._ + +_**Data type:** Object_ + +## Examples + +##### Calculate the length of geographic paths +```js +import "experimental/geo" + +region = { + minLat: 40.51757813, + maxLat: 40.86914063, + minLon: -73.65234375, + maxLon: -72.94921875 +} + +data + |> geo.toRows() + |> geo.asTracks() + |> geo.ST_LineString() + |> map(fn: (r) => ({ + r with st_length: geo.ST_Length(geometry: {linestring: r.st_linestring}) + })) +``` diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/st_linestring.md b/content/v2.0/reference/flux/stdlib/experimental/geo/st_linestring.md new file mode 100644 index 000000000..32d7a6715 --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/st_linestring.md @@ -0,0 +1,69 @@ +--- +title: geo.ST_LineString() function +description: > + The `geo.ST_LineString()` function converts a series of geographic points into + [linestring](/v2.0/reference/flux/stdlib/experimental/geo/#linestring). +menu: + v2_0_ref: + name: geo.ST_LineString + parent: Geo +weight: 401 +v2.0/tags: [functions, geo, GIS] +related: + - /v2.0/query-data/flux/geo/ +--- + +The `geo.ST_LineString()` function converts a series of geographic points into +[linestring](/v2.0/reference/flux/stdlib/experimental/geo/#linestring). +Group data into meaningful, ordered paths to before converting to linestring. +Rows in each table must have `lat` and `lon` columns. +Output tables contain a single row with a `st_linestring` column containing the resulting linestring. + +_**Function type:** Aggregate_ + +```js +import "experimental/geo" + +geo.ST_LineString() +``` + +## Examples + +### Convert a series of geographic points into linestring + +##### Input data + +| _time | id | lon | lat | +|:----- |:--: | ---: | ---: | +| 2020-01-01T00:00:00Z | a213b | 39.7515 | 14.01433 | +| 2020-01-02T00:00:00Z | a213b | 38.3527 | 13.9228 | +| 2020-01-03T00:00:00Z | a213b | 36.9978 | 15.08433 | + + +```js +import "experimental/geo" + +data + |> geo.ST_LineString() +``` + +##### Output data + +| id | st_linestring | +|:-- |:------------- | +| a213b | 39.7515 14.01433, 38.3527 13.9228, 36.9978 15.08433 | + +## Function definition +```js +ST_LineString = (tables=<-) => + tables + |> reduce(fn: (r, accumulator) => ({ + __linestring: accumulator.__linestring + (if accumulator.__count > 0 then ", " else "") + string(v: r.lat) + " " + string(v: r.lon), + __count: accumulator.__count + 1 + }), identity: { + __linestring: "", + __count: 0 + } + ) + |> rename(columns: {__linestring: "st_linestring"}) +``` diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/torows.md b/content/v2.0/reference/flux/stdlib/experimental/geo/torows.md index 57c3d07a0..7c6bb41b2 100644 --- a/content/v2.0/reference/flux/stdlib/experimental/geo/torows.md +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/torows.md @@ -22,19 +22,9 @@ _**Function type:** Transformation_ ```js import "experimental/geo" -geo.toRows( - correlationKey: ["_time"] -) +geo.toRows() ``` -## Parameters - -### correlationKey -List of columns used to uniquely identify a row for output. -Default is `["_time"]`. - -_**Data type:** Array of strings_ - ## Examples ```js import "experimental/geo" @@ -49,9 +39,5 @@ from(bucket: "example-bucket") ```js toRows = (tables=<-, correlationKey=["_time"]) => tables - |> pivot( - rowKey: correlationKey, - columnKey: ["_field"], - valueColumn: "_value" - ) + |> v1.fieldsAsCols() ``` diff --git a/content/v2.0/reference/release-notes/flux.md b/content/v2.0/reference/release-notes/flux.md index 777415d37..f254d2597 100644 --- a/content/v2.0/reference/release-notes/flux.md +++ b/content/v2.0/reference/release-notes/flux.md @@ -22,7 +22,8 @@ InfluxDB until the next InfluxDB v2.0 release._ - Add functions to convert semantic monotype to AST type. - Add BigQuery support. - Rust flatbuffer serialization for `MonoType` and `TypeExpression`. -- Extend with Geo package functions and add unit support. +- Extend with Geo package with GIS functions and + [unit support](/v2.0/reference/flux/stdlib/experimental/geo/#distance-units). ### Bug fixes - String interpolation in arrays. From 0861363f0c50a2f776608be23361b2cf0a099668 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 09:19:43 -0600 Subject: [PATCH 03/13] updated flux 0.78 release notes with breaking changes --- content/v2.0/reference/release-notes/flux.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/v2.0/reference/release-notes/flux.md b/content/v2.0/reference/release-notes/flux.md index f254d2597..5cb754610 100644 --- a/content/v2.0/reference/release-notes/flux.md +++ b/content/v2.0/reference/release-notes/flux.md @@ -18,6 +18,9 @@ InfluxDB until the next InfluxDB v2.0 release._ ## v0.78.0 [2020-08-10] +### Breaking changes +- Removed `correlationKey` parameter from `geo.toRows` and `geo.shapeData`. + ### Features - Add functions to convert semantic monotype to AST type. - Add BigQuery support. From ce71c90a2c44379f9cce8cb1ed6c7909dc45649b Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 10:36:29 -0600 Subject: [PATCH 04/13] fixed typo in geo index page --- content/v2.0/reference/flux/stdlib/experimental/geo/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md b/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md index 2a96d422e..963d23f46 100644 --- a/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md @@ -118,7 +118,7 @@ Define a circular region by specifying an object containing the following proper ``` ### point -Define a point region by specifying and object containing the following properties: +Define a point region by specifying an object containing the following properties: - **lat**: latitude in decimal degrees (WGS 84) _(Float)_ - **lon**: longitude in decimal degrees (WGS 84) _(Float)_ From 36cf187b203544d02e2ba83fdd910d99222857d7 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 14:18:15 -0600 Subject: [PATCH 05/13] updated geo docs to address PR feedback --- .../reference/flux/stdlib/experimental/geo/_index.md | 10 +++++++++- .../flux/stdlib/experimental/geo/st_dwithin.md | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md b/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md index 963d23f46..66806dfad 100644 --- a/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/_index.md @@ -74,6 +74,14 @@ to add `s2_cell_id` tags to data that includes fields with latitude and longitud ) ``` +## Latitude and longitude values +Flux supports latitude and longitude values in **decimal degrees** (WGS 84). + +| Coordinate | Minimum | Maximum | +|:---------- | -------:| -------:| +| Latitude | -90.0 | 90.0 | +| Longitude | -180.0 | 180.0 | + ## Region definitions Many functions in the Geo package filter data based on geographic region. Define geographic regions using the following shapes: @@ -162,7 +170,7 @@ Define GIS geometry using the following: ### linestring Define a geographic linestring path using an object containing the following properties: -- **linestring**: string containing comma-separatedlongitude and latitude +- **linestring**: string containing comma-separated longitude and latitude coordinate pairs (`lon lat,`): ```js diff --git a/content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md b/content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md index 4ee092ee4..011ba846f 100644 --- a/content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md +++ b/content/v2.0/reference/flux/stdlib/experimental/geo/st_dwithin.md @@ -49,7 +49,7 @@ _See [GIS geometry definitions](/v2.0/reference/flux/stdlib/experimental/geo/#gi _**Data type:** Object_ ### distance -Maximum allowed distance between the region and geometry. +Maximum distance allowed between the region and geometry. _Define distance units with the [`geo.units` option](/v2.0/reference/flux/stdlib/experimental/geo/#define-distance-units)._ _**Data type:** Float_ From eb329c7aba4d5bb2aab36bc0e7df02977e3475ee Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 16:33:50 -0600 Subject: [PATCH 06/13] added bigquery to sql package --- .../v2.0/reference/flux/stdlib/sql/_index.md | 4 +- .../v2.0/reference/flux/stdlib/sql/from.md | 45 ++++++++++++++++ content/v2.0/reference/flux/stdlib/sql/to.md | 53 +++++++++++++++++-- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/content/v2.0/reference/flux/stdlib/sql/_index.md b/content/v2.0/reference/flux/stdlib/sql/_index.md index 53dadf6c5..f2fbc92d5 100644 --- a/content/v2.0/reference/flux/stdlib/sql/_index.md +++ b/content/v2.0/reference/flux/stdlib/sql/_index.md @@ -3,7 +3,8 @@ title: Flux SQL package list_title: SQL package description: > The Flux SQL package provides tools for working with data in SQL databases such - as MySQL, PostgreSQL, Snowflake, SQLite, Microsoft SQL Server, and Amazon Athena. + as MySQL, PostgreSQL, Snowflake, SQLite, Microsoft SQL Server, Amazon Athena, + and Google BigQuery. Import the `sql` package. aliases: - /v2.0/reference/flux/functions/sql/ @@ -20,6 +21,7 @@ related: SQL Flux functions provide tools for working with data in SQL databases such as: - Amazon Athena +- Google BigQuery - Microsoft SQL Server - MySQL - PostgreSQL diff --git a/content/v2.0/reference/flux/stdlib/sql/from.md b/content/v2.0/reference/flux/stdlib/sql/from.md index f48d16a92..69a5437b9 100644 --- a/content/v2.0/reference/flux/stdlib/sql/from.md +++ b/content/v2.0/reference/flux/stdlib/sql/from.md @@ -36,6 +36,7 @@ _**Data type:** String_ The following drivers are available: - awsathena +- bigquery - mysql - postgres - snowflake @@ -73,6 +74,10 @@ sqlserver://username:password@localhost:1234?database=examplebdb server=localhost;user id=username;database=examplebdb; server=localhost;user id=username;database=examplebdb;azure auth=ENV server=localhost;user id=username;database=examplebdbr;azure tenant id=77e7d537;azure client id=58879ce8;azure client secret=0123456789 + +# Google BigQuery DSNs +bigquery://projectid/?param1=value¶m2=value +bigquery://projectid/location?param1=value¶m2=value ``` ### query @@ -88,6 +93,7 @@ _**Data type:** String_ - [SQLite](#query-an-sqlite-database) - [Amazon Athena](#query-an-amazon-athena-database) - [SQL Server](#query-a-sql-server-database) +- [Google BigQuery](#query-a-bigquery-database) {{% note %}} The examples below use [InfluxDB secrets](/v2.0/security/secrets/) to populate @@ -250,3 +256,42 @@ _For information about managed identities, see [Microsoft managed identities](ht ``` azure auth=MSI ``` + +### Query a BigQuery database +```js +import "sql" +import "influxdata/influxdb/secrets" + +projectID = secrets.get(key: "BIGQUERY_PROJECT_ID") +apiKey = secrets.get(key: "BIGQUERY_APIKEY") + +sql.from( + driverName: "bigquery", + dataSourceName: "bigquery://${projectID}/?apiKey=${apiKey}", + query:"SELECT * FROM exampleTable" +) +``` + +#### Common BigQuery URL parameters +- **dataset** - BigQuery dataset ID. When set, you can use unqualified tables names in queries. + +#### BigQuery authentication parameters +The Flux BigQuery implementation uses the Google Cloud Go SDK. +Provide your authentication credentials using one of the following methods: + +- The `GOOGLE_APPLICATION_CREDENTIALS` environment variable that identifies the + location of your credential JSON file. +- Provide your BigQuery API key using the **apiKey** URL parameter in your BigQuery DSN. + + ###### Example apiKey URL parameter + ``` + bigquery://projectid/?apiKey=AIzaSyB6XK8IO5AzKZXoioQOVNTFYzbDBjY5hy4 + ``` + +- Provide your base-64 encoded service account, refresh token, or JSON credentials + using the **credentials** URL parameter in your BigQuery DSN. + + ###### Example credentials URL parameter + ``` + bigquery://projectid/?credentials=eyJ0eXBlIjoiYXV0... + ``` diff --git a/content/v2.0/reference/flux/stdlib/sql/to.md b/content/v2.0/reference/flux/stdlib/sql/to.md index 0dc69b006..77b36ad15 100644 --- a/content/v2.0/reference/flux/stdlib/sql/to.md +++ b/content/v2.0/reference/flux/stdlib/sql/to.md @@ -34,12 +34,18 @@ _**Data type:** String_ The following drivers are available: +- bigquery - mysql - postgres - snowflake - sqlite3 – _Does not work with InfluxDB OSS or InfluxDB Cloud. More information [below](#write-data-to-an-sqlite-database)._ - sqlserver, mssql +{{% warn %}} +### sql.to does not support Amazon Athena +The `sql.to` function does not support writing data to [Amazon Athena](https://aws.amazon.com/athena/). +{{% /warn %}} + ### dataSourceName The data source name (DSN) or connection string used to connect to the SQL database. The string's form and structure depend on the [driver](#drivername) used. @@ -67,6 +73,10 @@ sqlserver://username:password@localhost:1234?database=examplebdb server=localhost;user id=username;database=examplebdb; server=localhost;user id=username;database=examplebdb;azure auth=ENV server=localhost;user id=username;database=examplebdbr;azure tenant id=77e7d537;azure client id=58879ce8;azure client secret=0123456789 + +# Google BigQuery DSNs +bigquery://projectid/?param1=value¶m2=value +bigquery://projectid/location?param1=value¶m2=value ``` ### table @@ -91,6 +101,7 @@ If writing to a **SQLite** database, set `batchSize` to `999` or less. - [Snowflake](#write-data-to-a-snowflake-database) - [SQLite](#write-data-to-an-sqlite-database) - [SQL Server](#write-data-to-a-sql-server-database) +- [Google BigQuery](#write-data-to-a-sql-server-database) {{% note %}} The examples below use [InfluxDB secrets](/v2.0/security/secrets/) to populate @@ -223,7 +234,41 @@ _For information about managed identities, see [Microsoft managed identities](ht azure auth=MSI ``` -{{% warn %}} -### sql.to does not support Amazon Athena -The `sql.to` function does not support writing data to [Amazon Athena](https://aws.amazon.com/athena/). -{{% /warn %}} +### Write to a BigQuery database +```js +import "sql" +import "influxdata/influxdb/secrets" + +projectID = secrets.get(key: "BIGQUERY_PROJECT_ID") +apiKey = secrets.get(key: "BIGQUERY_APIKEY") + +sql.to( + driverName: "bigquery", + dataSourceName: "bigquery://${projectID}/?apiKey=${apiKey}", + table:"exampleTable" +) +``` + +#### Common BigQuery URL parameters +- **dataset** - BigQuery dataset ID. When set, you can use unqualified tables names in queries. + +#### BigQuery authentication parameters +The Flux BigQuery implementation uses the Google Cloud Go SDK. +Provide your authentication credentials using one of the following methods: + +- The `GOOGLE_APPLICATION_CREDENTIALS` environment variable that identifies the + location of your credential JSON file. +- Provide your BigQuery API key using the **apiKey** URL parameter in your BigQuery DSN. + + ###### Example apiKey URL parameter + ``` + bigquery://projectid/?apiKey=AIzaSyB6XK8IO5AzKZXoioQOVNTFYzbDBjY5hy4 + ``` + +- Provide your base-64 encoded service account, refresh token, or JSON credentials + using the **credentials** URL parameter in your BigQuery DSN. + + ###### Example credentials URL parameter + ``` + bigquery://projectid/?credentials=eyJ0eXBlIjoiYXV0... + ``` From ea1acd43fef9608b73826761a317dd4b5134a782 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 16:48:01 -0600 Subject: [PATCH 07/13] added bigquery to query sql db guide --- content/v2.0/query-data/flux/sql.md | 34 +++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/content/v2.0/query-data/flux/sql.md b/content/v2.0/query-data/flux/sql.md index 850d0a074..311259c6b 100644 --- a/content/v2.0/query-data/flux/sql.md +++ b/content/v2.0/query-data/flux/sql.md @@ -33,8 +33,8 @@ The [Flux](/v2.0/reference/flux) `sql` package provides functions for working wi like [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/), [Snowflake](https://www.snowflake.com/), [SQLite](https://www.sqlite.org/index.html), [Microsoft SQL Server](https://www.microsoft.com/en-us/sql-server/default.aspx), -and [Amazon Athena](https://aws.amazon.com/athena/) and use the results with -InfluxDB dashboards, tasks, and other operations. +[Amazon Athena](https://aws.amazon.com/athena/) and [Google BigQuery](https://cloud.google.com/bigquery) +and use the results with InfluxDB dashboards, tasks, and other operations. - [Query a SQL data source](#query-a-sql-data-source) - [Join SQL data with data in InfluxDB](#join-sql-data-with-data-in-influxdb) @@ -61,6 +61,8 @@ To query a SQL data source: [Snowflake](#) [SQLite](#) [SQL Server](#) +[Athena](#) +[BigQuery](#) {{% /code-tabs %}} {{% code-tab-content %}} @@ -129,6 +131,34 @@ sql.from( _For information about authenticating with SQL Server using ADO-style parameters, see [SQL Server ADO authentication](/v2.0/reference/flux/stdlib/sql/from/#sql-server-ado-authentication)._ {{% /code-tab-content %}} +{{% code-tab-content %}} +```js +import "sql" + +sql.from( + driverName: "awsathena", + dataSourceName: "s3://myorgqueryresults/?accessID=12ab34cd56ef®ion=region-name&secretAccessKey=y0urSup3rs3crEtT0k3n", + query: "GO SELECT * FROM Example.Table" +) +``` + +_For information about parameters to include in the Athena DSN, +see [Athena connection string](/v2.0/reference/flux/stdlib/sql/from/#athena-connection-string)._ +{{% /code-tab-content %}} +{{% code-tab-content %}} +```js +import "sql" + +sql.from( + driverName: "bigquery", + dataSourceName: "bigquery://projectid/?apiKey=mySuP3r5ecR3tAP1K3y", + query: "SELECT * FROM exampleTable" +) +``` + +_For information about authenticating with BigQuery, see +[BigQuery authentication parameters](/v2.0/reference/flux/stdlib/sql/from/#bigquery-authentication-parameters)._ +{{% /code-tab-content %}} {{< /code-tabs-wrapper >}} _See the [`sql.from()` documentation](/v2.0/reference/flux/stdlib/sql/from/) for From 9571096c5f9d94a8c69bea1aef8afd5ab0689f53 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 17:02:21 -0600 Subject: [PATCH 08/13] added bigquery links to flux changelog --- content/v2.0/reference/release-notes/flux.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/v2.0/reference/release-notes/flux.md b/content/v2.0/reference/release-notes/flux.md index 5cb754610..120db7b41 100644 --- a/content/v2.0/reference/release-notes/flux.md +++ b/content/v2.0/reference/release-notes/flux.md @@ -23,7 +23,8 @@ InfluxDB until the next InfluxDB v2.0 release._ ### Features - Add functions to convert semantic monotype to AST type. -- Add BigQuery support. +- Add [Google BigQuery](https://cloud.google.com/bigquery) support to + [SQL package](/v2.0/reference/flux/stdlib/sql/). - Rust flatbuffer serialization for `MonoType` and `TypeExpression`. - Extend with Geo package with GIS functions and [unit support](/v2.0/reference/flux/stdlib/experimental/geo/#distance-units). From fe4fd7967e3fd37471d70ec6fff1b9992cbd9302 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 22:49:23 -0600 Subject: [PATCH 09/13] quick typo fix in sql.from doc --- content/v2.0/reference/flux/stdlib/sql/from.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/v2.0/reference/flux/stdlib/sql/from.md b/content/v2.0/reference/flux/stdlib/sql/from.md index 69a5437b9..514344f7a 100644 --- a/content/v2.0/reference/flux/stdlib/sql/from.md +++ b/content/v2.0/reference/flux/stdlib/sql/from.md @@ -273,7 +273,7 @@ sql.from( ``` #### Common BigQuery URL parameters -- **dataset** - BigQuery dataset ID. When set, you can use unqualified tables names in queries. +- **dataset** - BigQuery dataset ID. When set, you can use unqualified table names in queries. #### BigQuery authentication parameters The Flux BigQuery implementation uses the Google Cloud Go SDK. From db00320508c42a74b6e8e6c62ba5b06eb186bc30 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 22:49:43 -0600 Subject: [PATCH 10/13] quick typo fix in sql.to doc --- content/v2.0/reference/flux/stdlib/sql/to.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/v2.0/reference/flux/stdlib/sql/to.md b/content/v2.0/reference/flux/stdlib/sql/to.md index 77b36ad15..4c86a4124 100644 --- a/content/v2.0/reference/flux/stdlib/sql/to.md +++ b/content/v2.0/reference/flux/stdlib/sql/to.md @@ -250,7 +250,7 @@ sql.to( ``` #### Common BigQuery URL parameters -- **dataset** - BigQuery dataset ID. When set, you can use unqualified tables names in queries. +- **dataset** - BigQuery dataset ID. When set, you can use unqualified table names in queries. #### BigQuery authentication parameters The Flux BigQuery implementation uses the Google Cloud Go SDK. From f85f76310e1412e51f5c6dfd3f3d191f4e22aa81 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 23:21:33 -0600 Subject: [PATCH 11/13] Revert "Flux Google BigQuery support" --- content/v2.0/query-data/flux/sql.md | 34 +----------- .../v2.0/reference/flux/stdlib/sql/_index.md | 4 +- .../v2.0/reference/flux/stdlib/sql/from.md | 45 ---------------- content/v2.0/reference/flux/stdlib/sql/to.md | 53 ++----------------- content/v2.0/reference/release-notes/flux.md | 3 +- 5 files changed, 8 insertions(+), 131 deletions(-) diff --git a/content/v2.0/query-data/flux/sql.md b/content/v2.0/query-data/flux/sql.md index 311259c6b..850d0a074 100644 --- a/content/v2.0/query-data/flux/sql.md +++ b/content/v2.0/query-data/flux/sql.md @@ -33,8 +33,8 @@ The [Flux](/v2.0/reference/flux) `sql` package provides functions for working wi like [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/), [Snowflake](https://www.snowflake.com/), [SQLite](https://www.sqlite.org/index.html), [Microsoft SQL Server](https://www.microsoft.com/en-us/sql-server/default.aspx), -[Amazon Athena](https://aws.amazon.com/athena/) and [Google BigQuery](https://cloud.google.com/bigquery) -and use the results with InfluxDB dashboards, tasks, and other operations. +and [Amazon Athena](https://aws.amazon.com/athena/) and use the results with +InfluxDB dashboards, tasks, and other operations. - [Query a SQL data source](#query-a-sql-data-source) - [Join SQL data with data in InfluxDB](#join-sql-data-with-data-in-influxdb) @@ -61,8 +61,6 @@ To query a SQL data source: [Snowflake](#) [SQLite](#) [SQL Server](#) -[Athena](#) -[BigQuery](#) {{% /code-tabs %}} {{% code-tab-content %}} @@ -131,34 +129,6 @@ sql.from( _For information about authenticating with SQL Server using ADO-style parameters, see [SQL Server ADO authentication](/v2.0/reference/flux/stdlib/sql/from/#sql-server-ado-authentication)._ {{% /code-tab-content %}} -{{% code-tab-content %}} -```js -import "sql" - -sql.from( - driverName: "awsathena", - dataSourceName: "s3://myorgqueryresults/?accessID=12ab34cd56ef®ion=region-name&secretAccessKey=y0urSup3rs3crEtT0k3n", - query: "GO SELECT * FROM Example.Table" -) -``` - -_For information about parameters to include in the Athena DSN, -see [Athena connection string](/v2.0/reference/flux/stdlib/sql/from/#athena-connection-string)._ -{{% /code-tab-content %}} -{{% code-tab-content %}} -```js -import "sql" - -sql.from( - driverName: "bigquery", - dataSourceName: "bigquery://projectid/?apiKey=mySuP3r5ecR3tAP1K3y", - query: "SELECT * FROM exampleTable" -) -``` - -_For information about authenticating with BigQuery, see -[BigQuery authentication parameters](/v2.0/reference/flux/stdlib/sql/from/#bigquery-authentication-parameters)._ -{{% /code-tab-content %}} {{< /code-tabs-wrapper >}} _See the [`sql.from()` documentation](/v2.0/reference/flux/stdlib/sql/from/) for diff --git a/content/v2.0/reference/flux/stdlib/sql/_index.md b/content/v2.0/reference/flux/stdlib/sql/_index.md index f2fbc92d5..53dadf6c5 100644 --- a/content/v2.0/reference/flux/stdlib/sql/_index.md +++ b/content/v2.0/reference/flux/stdlib/sql/_index.md @@ -3,8 +3,7 @@ title: Flux SQL package list_title: SQL package description: > The Flux SQL package provides tools for working with data in SQL databases such - as MySQL, PostgreSQL, Snowflake, SQLite, Microsoft SQL Server, Amazon Athena, - and Google BigQuery. + as MySQL, PostgreSQL, Snowflake, SQLite, Microsoft SQL Server, and Amazon Athena. Import the `sql` package. aliases: - /v2.0/reference/flux/functions/sql/ @@ -21,7 +20,6 @@ related: SQL Flux functions provide tools for working with data in SQL databases such as: - Amazon Athena -- Google BigQuery - Microsoft SQL Server - MySQL - PostgreSQL diff --git a/content/v2.0/reference/flux/stdlib/sql/from.md b/content/v2.0/reference/flux/stdlib/sql/from.md index 514344f7a..f48d16a92 100644 --- a/content/v2.0/reference/flux/stdlib/sql/from.md +++ b/content/v2.0/reference/flux/stdlib/sql/from.md @@ -36,7 +36,6 @@ _**Data type:** String_ The following drivers are available: - awsathena -- bigquery - mysql - postgres - snowflake @@ -74,10 +73,6 @@ sqlserver://username:password@localhost:1234?database=examplebdb server=localhost;user id=username;database=examplebdb; server=localhost;user id=username;database=examplebdb;azure auth=ENV server=localhost;user id=username;database=examplebdbr;azure tenant id=77e7d537;azure client id=58879ce8;azure client secret=0123456789 - -# Google BigQuery DSNs -bigquery://projectid/?param1=value¶m2=value -bigquery://projectid/location?param1=value¶m2=value ``` ### query @@ -93,7 +88,6 @@ _**Data type:** String_ - [SQLite](#query-an-sqlite-database) - [Amazon Athena](#query-an-amazon-athena-database) - [SQL Server](#query-a-sql-server-database) -- [Google BigQuery](#query-a-bigquery-database) {{% note %}} The examples below use [InfluxDB secrets](/v2.0/security/secrets/) to populate @@ -256,42 +250,3 @@ _For information about managed identities, see [Microsoft managed identities](ht ``` azure auth=MSI ``` - -### Query a BigQuery database -```js -import "sql" -import "influxdata/influxdb/secrets" - -projectID = secrets.get(key: "BIGQUERY_PROJECT_ID") -apiKey = secrets.get(key: "BIGQUERY_APIKEY") - -sql.from( - driverName: "bigquery", - dataSourceName: "bigquery://${projectID}/?apiKey=${apiKey}", - query:"SELECT * FROM exampleTable" -) -``` - -#### Common BigQuery URL parameters -- **dataset** - BigQuery dataset ID. When set, you can use unqualified table names in queries. - -#### BigQuery authentication parameters -The Flux BigQuery implementation uses the Google Cloud Go SDK. -Provide your authentication credentials using one of the following methods: - -- The `GOOGLE_APPLICATION_CREDENTIALS` environment variable that identifies the - location of your credential JSON file. -- Provide your BigQuery API key using the **apiKey** URL parameter in your BigQuery DSN. - - ###### Example apiKey URL parameter - ``` - bigquery://projectid/?apiKey=AIzaSyB6XK8IO5AzKZXoioQOVNTFYzbDBjY5hy4 - ``` - -- Provide your base-64 encoded service account, refresh token, or JSON credentials - using the **credentials** URL parameter in your BigQuery DSN. - - ###### Example credentials URL parameter - ``` - bigquery://projectid/?credentials=eyJ0eXBlIjoiYXV0... - ``` diff --git a/content/v2.0/reference/flux/stdlib/sql/to.md b/content/v2.0/reference/flux/stdlib/sql/to.md index 4c86a4124..0dc69b006 100644 --- a/content/v2.0/reference/flux/stdlib/sql/to.md +++ b/content/v2.0/reference/flux/stdlib/sql/to.md @@ -34,18 +34,12 @@ _**Data type:** String_ The following drivers are available: -- bigquery - mysql - postgres - snowflake - sqlite3 – _Does not work with InfluxDB OSS or InfluxDB Cloud. More information [below](#write-data-to-an-sqlite-database)._ - sqlserver, mssql -{{% warn %}} -### sql.to does not support Amazon Athena -The `sql.to` function does not support writing data to [Amazon Athena](https://aws.amazon.com/athena/). -{{% /warn %}} - ### dataSourceName The data source name (DSN) or connection string used to connect to the SQL database. The string's form and structure depend on the [driver](#drivername) used. @@ -73,10 +67,6 @@ sqlserver://username:password@localhost:1234?database=examplebdb server=localhost;user id=username;database=examplebdb; server=localhost;user id=username;database=examplebdb;azure auth=ENV server=localhost;user id=username;database=examplebdbr;azure tenant id=77e7d537;azure client id=58879ce8;azure client secret=0123456789 - -# Google BigQuery DSNs -bigquery://projectid/?param1=value¶m2=value -bigquery://projectid/location?param1=value¶m2=value ``` ### table @@ -101,7 +91,6 @@ If writing to a **SQLite** database, set `batchSize` to `999` or less. - [Snowflake](#write-data-to-a-snowflake-database) - [SQLite](#write-data-to-an-sqlite-database) - [SQL Server](#write-data-to-a-sql-server-database) -- [Google BigQuery](#write-data-to-a-sql-server-database) {{% note %}} The examples below use [InfluxDB secrets](/v2.0/security/secrets/) to populate @@ -234,41 +223,7 @@ _For information about managed identities, see [Microsoft managed identities](ht azure auth=MSI ``` -### Write to a BigQuery database -```js -import "sql" -import "influxdata/influxdb/secrets" - -projectID = secrets.get(key: "BIGQUERY_PROJECT_ID") -apiKey = secrets.get(key: "BIGQUERY_APIKEY") - -sql.to( - driverName: "bigquery", - dataSourceName: "bigquery://${projectID}/?apiKey=${apiKey}", - table:"exampleTable" -) -``` - -#### Common BigQuery URL parameters -- **dataset** - BigQuery dataset ID. When set, you can use unqualified table names in queries. - -#### BigQuery authentication parameters -The Flux BigQuery implementation uses the Google Cloud Go SDK. -Provide your authentication credentials using one of the following methods: - -- The `GOOGLE_APPLICATION_CREDENTIALS` environment variable that identifies the - location of your credential JSON file. -- Provide your BigQuery API key using the **apiKey** URL parameter in your BigQuery DSN. - - ###### Example apiKey URL parameter - ``` - bigquery://projectid/?apiKey=AIzaSyB6XK8IO5AzKZXoioQOVNTFYzbDBjY5hy4 - ``` - -- Provide your base-64 encoded service account, refresh token, or JSON credentials - using the **credentials** URL parameter in your BigQuery DSN. - - ###### Example credentials URL parameter - ``` - bigquery://projectid/?credentials=eyJ0eXBlIjoiYXV0... - ``` +{{% warn %}} +### sql.to does not support Amazon Athena +The `sql.to` function does not support writing data to [Amazon Athena](https://aws.amazon.com/athena/). +{{% /warn %}} diff --git a/content/v2.0/reference/release-notes/flux.md b/content/v2.0/reference/release-notes/flux.md index 120db7b41..5cb754610 100644 --- a/content/v2.0/reference/release-notes/flux.md +++ b/content/v2.0/reference/release-notes/flux.md @@ -23,8 +23,7 @@ InfluxDB until the next InfluxDB v2.0 release._ ### Features - Add functions to convert semantic monotype to AST type. -- Add [Google BigQuery](https://cloud.google.com/bigquery) support to - [SQL package](/v2.0/reference/flux/stdlib/sql/). +- Add BigQuery support. - Rust flatbuffer serialization for `MonoType` and `TypeExpression`. - Extend with Geo package with GIS functions and [unit support](/v2.0/reference/flux/stdlib/experimental/geo/#distance-units). From 71c15534bb303033743b126fcdfe25251d59f20f Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 23:26:22 -0600 Subject: [PATCH 12/13] added flux 0.79.0 release notes --- content/v2.0/reference/release-notes/flux.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/content/v2.0/reference/release-notes/flux.md b/content/v2.0/reference/release-notes/flux.md index 5cb754610..7be27cda0 100644 --- a/content/v2.0/reference/release-notes/flux.md +++ b/content/v2.0/reference/release-notes/flux.md @@ -16,6 +16,16 @@ Though newer versions of Flux may be available, they will not be included with InfluxDB until the next InfluxDB v2.0 release._ {{% /note %}} +## v0.79.0 [2020-08-11] + +### Features +- Add `array.from()` function to convert Flux values into a table. + +### Bug fixes +- Add bounds to Geo package end-to-end tests. + +--- + ## v0.78.0 [2020-08-10] ### Breaking changes From 9944fc4d3bd6f6acac55fd107a4e4306d9aff6b8 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Tue, 11 Aug 2020 23:59:42 -0600 Subject: [PATCH 13/13] added flux array package --- .../flux/stdlib/experimental/array/_index.md | 22 +++++++ .../flux/stdlib/experimental/array/from.md | 63 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 content/v2.0/reference/flux/stdlib/experimental/array/_index.md create mode 100644 content/v2.0/reference/flux/stdlib/experimental/array/from.md diff --git a/content/v2.0/reference/flux/stdlib/experimental/array/_index.md b/content/v2.0/reference/flux/stdlib/experimental/array/_index.md new file mode 100644 index 000000000..56b73c88d --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/array/_index.md @@ -0,0 +1,22 @@ +--- +title: Flux Array package +list_title: Array package +description: > + The Flux Array package provides functions for building tables from Flux arrays. + Import the `experimental/array` package. +menu: + v2_0_ref: + name: Array + parent: Experimental +weight: 301 +v2.0/tags: [functions, array, package, table] +--- + +Flux Array functions provide tools for building tables from Flux arrays. +Import the `experimental/array` package: + +```js +import "experimental/array" +``` + +{{< children type="functions" show="pages" >}} diff --git a/content/v2.0/reference/flux/stdlib/experimental/array/from.md b/content/v2.0/reference/flux/stdlib/experimental/array/from.md new file mode 100644 index 000000000..3bf188488 --- /dev/null +++ b/content/v2.0/reference/flux/stdlib/experimental/array/from.md @@ -0,0 +1,63 @@ +--- +title: array.from() function +description: > + The experimental `array.from()` function constructs a table from an array of objects. +menu: + v2_0_ref: + name: array.from + parent: Array +weight: 401 +--- + +The experimental `array.from()` function constructs a table from an array of objects. +Each object in the array is converted into an output row or record. +All records must have the same keys and data types. + +_**Function type:** Input_ + +{{< keep-url >}} +```js +import "experimental/array" + +array.from(rows: [ + {_time: 2020-01-01T00:00:00Z, _field: "exampleField", _value: 3, foo: "bar"}, + {_time: 2020-01-01T00:01:00Z, _field: "exampleField", _value: 4, foo: "bar"}, + {_time: 2020-01-01T00:02:00Z, _field: "exampleField", _value: 1, foo: "bar"} +]) +``` + +## Parameters + +### rows +Array of records to construct a table with. + +_**Data type:** Array of objects_ + +## Examples + +##### Build an arbitrary table +```js +import "experimental/array" + +rows = [ + {foo: "bar", baz: 21.2} + {foo: "bar", baz: 23.8} +] + +array.from(rows: rows) +``` + +##### Union custom rows with query results +```js +import "influxdata/influxdb/v1" +import "experimental/array" + +tags = v1.tagValues( + bucket: "example-bucket", + tag: "host" +) + +wildcard_tag = array.from(rows: [{_value: "*"}]) + +union(tables: [tags, wildcard_tag]) +```