From b678ef583c214bfcc1560620495b500a78b2511d Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Thu, 26 Mar 2020 16:03:51 -0600 Subject: [PATCH] added linepart annotation to annotated csv doc, resolves #857 --- .../v2.0/reference/syntax/annotated-csv.md | 133 +++++++++++++----- 1 file changed, 94 insertions(+), 39 deletions(-) diff --git a/content/v2.0/reference/syntax/annotated-csv.md b/content/v2.0/reference/syntax/annotated-csv.md index 854be0c2d..4e0580d76 100644 --- a/content/v2.0/reference/syntax/annotated-csv.md +++ b/content/v2.0/reference/syntax/annotated-csv.md @@ -2,8 +2,9 @@ title: Annotated CSV syntax list_title: Annotated CSV description: > - Flux returns raw results in Annotated CSV format and also reads Annotated CSV - using the `csv.from()` function. + InfluxDB and Flux return query results in Annotated CSV format. + You can also read annotated CSV directly from Flux with the `csv.from()` function + or write data to InfluxDB using annotated CSV and the `influx write` command. weight: 103 menu: v2_0_ref: @@ -14,8 +15,9 @@ aliases: - /v2.0/reference/annotated-csv/ --- -Flux returns raw results in Annotated CSV format and also reads Annotated CSV -using the [`csv.from()` function](/v2.0/reference/flux/stdlib/csv/from/). +InfluxDB and Flux return query results in Annotated CSV format. +You can also read annotated CSV directly from Flux with the [`csv.from()` function](/v2.0/reference/flux/stdlib/csv/from/) +or write data to InfluxDB using annotated CSV and the `influx write` command. CSV tables must be encoded in UTF-8 and Unicode Normal Form C as defined in [UAX15](http://www.unicode.org/reports/tr15/). Line endings must be CRLF (Carriage Return Line Feed) as defined by the `text/csv` @@ -59,7 +61,7 @@ Encoding of a table with and without a header row. {{% /code-tabs %}} {{% code-tab-content %}} -```js +```sh result,table,_start,_stop,_time,region,host,_value my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,east,A,15.43 my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,east,B,59.25 @@ -68,7 +70,7 @@ my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,east, {{% /code-tab-content %}} {{% code-tab-content %}} -```js +```sh my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,east,A,15.43 my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,east,B,59.25 my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,east,C,52.62 @@ -111,7 +113,7 @@ Encoding of two tables in the same result with the same schema (header row) and {{% /code-tabs %}} {{% code-tab-content %}} -```js +```sh result,table,_start,_stop,_time,region,host,_value my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,east,A,15.43 my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,east,B,59.25 @@ -124,7 +126,7 @@ my-result,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,west, {{% /code-tab-content %}} {{% code-tab-content %}} -```js +```sh ,result,table,_start,_stop,_time,region,host,_value ,my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,east,A,15.43 ,my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,east,B,59.25 @@ -138,7 +140,7 @@ my-result,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,west, {{% /code-tab-content %}} {{< /code-tabs-wrapper >}} -### Dialect options +## Dialect options Flux supports the following dialect options for `text/csv` format. @@ -150,25 +152,26 @@ Flux supports the following dialect options for `text/csv` format. | **annotations** | List of annotations to encode (datatype, group, or default). | `empty` | | **commentPrefix** | String prefix to identify a comment. Always added to annotations. | `#` | -### Annotations +## Annotations Annotation rows describe column properties, and start with `#` (or commentPrefix value). The first column in an annotation row always contains the annotation name. Subsequent columns contain annotation values as shown in the table below. -| Annotation name | Values | Description | -|:-------- |:--------- | :------- | -| **datatype** | a [valid data type](#valid-data-types) | Describes the type of data. | -| **group** | boolean flag `true` or `false` | Indicates the column is part of the group key. | -| **default** | a [valid data type](#valid-data-types) | Value to use for rows with an empty string value. | +| Annotation name | Values | Description | +|:-------- |:--------- | :------- | +| **datatype** | a [valid data type](#valid-data-types) | Describes the type of data. | +| **group** | boolean flag `true` or `false` | Indicates the column is part of the group key. | +| **default** | a [valid data type](#valid-data-types) | Value to use for rows with an empty string value. | +| **linepart** | a [valid line protocol element](#line-protocol-and-linepart-annotations) | Indicates how to parse the column into line protocol. | {{% note %}} -To encode a table with its group key, the `datatype`, `group`, and `default` annotations must be included. If a table has no rows, the `default` annotation provides the group key values. +To encode a table with its group key, the `datatype`, `group`, and `default` annotations must be included. +If a table has no rows, the `default` annotation provides the group key values. {{% /note %}} -##### Example - -Example encoding of datatype, group, and default annotations. +### Annotated CSV in a Flux query +Example encoding of datatype, group, and default annotations for using annotated CSV with Flux: ```js import "csv" @@ -177,29 +180,81 @@ csvData = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3 #group,false,false,false,false,false,false,false,false #default,,,,,,,, ,result,table,_start,_stop,_time,region,host,_value -,my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,east,A,15.43 -,my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,east,B,59.25 -,my-result,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,east,C,52.62 -,my-result,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,west,A,62.73 -,my-result,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,west,B,12.83 -,my-result,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,west,C,51.62 +,,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,east,A,15.43 +,,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,east,B,59.25 +,,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,east,C,52.62 +,,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,west,A,62.73 +,,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,west,B,12.83 +,,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,west,C,51.62 " csv.from(csv: csvData) ``` -### Valid data types +### Annotated CSV with linepart annotations +The `linepart` annotation tells the [`influx write` command](/v2.0/reference/cli/influx/write/) +how to parse the annotated CSV into line protocol when writing to InfluxDB. -| Datatype | Flux type | Description | -| :-------- | :--------- | :---------- | -| boolean | bool | a truth value, one of "true" or "false" | -| unsignedLong | uint | an unsigned 64-bit integer | -| long | int | a signed 64-bit integer | -| double | float | an IEEE-754 64-bit floating-point number | -| string | string | a UTF-8 encoded string | -| base64Binary | bytes | a base64 encoded sequence of bytes as defined in RFC 4648 | -| dateTime | time | an instant in time, may be followed with a colon : and a description of the format | -| duration | duration | a length of time represented as an unsigned 64-bit integer number of nanoseconds | +``` +#linepart measurement,tag,tag,field,field,ignored,time +m,cpu,host,time_steal,usage_user,nothing,time +cpu,cpu1,host1,0,2.7,a,1482669077000000000 +cpu,cpu1,host2,0,2.2,b,1482669087000000000 +``` + +#### Line protocol and linepart annotations +| Linepart key | Description | +|:------------ |:----------- | +| `measurement` | column value parsed as the measurement | +| `field` (default) | column header parsed as field key, column value parsed as field value | +| `tag` | column header parsed as tag key, column value parsed as tag value | +| `time` | column value parsed as timestamp | +| `ignore` or`ignored` | column is not included in line protocol | + +##### Example linepart annotations +``` +#linepart measurement,tag,tag,field,field,ignored,time +m,cpu,host,time_steal,usage_user,nothing,time +cpu,cpu1,host1,0,2.7,a,1482669077000000000 +cpu,cpu1,host2,0,2.2,b,1482669087000000000 +``` + +Resulting line protocol: + +``` +cpu,cpu=cpu1,host=host1 time_steal=0,usage_user=2.7 1482669077000000000 +cpu,cpu=cpu1,host=host1 time_steal=0,usage_user=2.2 1482669087000000000 +``` + +##### Example linepart, datatype, and default annotations +``` +#datatype ,,string,double,boolean,long,unsignedLong,duration, +#linepart measurement,tag,,,,,,,time +#default test,annotatedDatatypes,,,,,, +m,name,s,d,b,l,ul,dur,time +,,str1,1.0,true,1,1,1ms,1 +,,str2,2.0,false,2,2,2us,2020-01-11T10:10:10Z +``` + +Resulting line protocol: + +``` +test,name=annotatedDatatypes s="str1",d=1,b=true,l=1i,ul=1u,dur=1000000i 1 +test,name=annotatedDatatypes s="str2",d=2,b=false,l=2i,ul=2u,dur=2000i 1578737410000000000 +``` + +## Valid data types + +| Datatype | Flux type | Description | +| :-------- | :--------- | :---------- | +| boolean | bool | a truth value, one of "true" or "false" | +| unsignedLong | uint | an unsigned 64-bit integer | +| long | int | a signed 64-bit integer | +| double | float | an IEEE-754 64-bit floating-point number | +| string | string | a UTF-8 encoded string | +| base64Binary | bytes | a base64 encoded sequence of bytes as defined in RFC 4648 | +| dateTime | time | an instant in time, may be followed with a colon : and a description of the format (RFC3339, RFC3339Nano) | +| duration | duration | a length of time represented as an unsigned 64-bit integer number of nanoseconds | ## Errors @@ -218,7 +273,7 @@ If an error occurs: Encoding for an error with the datatype annotation: -```csv +``` #datatype,string,long ,error,reference ,Failed to parse query,897 @@ -226,7 +281,7 @@ Encoding for an error with the datatype annotation: Encoding for an error that occurs after a valid table has been encoded: -```csv +``` #datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,string,string,double ,result,table,_start,_stop,_time,region,host,_value ,my-result,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,west,A,62.73 @@ -234,7 +289,7 @@ Encoding for an error that occurs after a valid table has been encoded: ,my-result,1,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,west,C,51.62 ``` -```csv +``` #datatype,string,long ,error,reference,query terminated: reached maximum allowed memory limits,576 ```