339 lines
9.7 KiB
Go
339 lines
9.7 KiB
Go
package write
|
|
|
|
import (
|
|
"encoding/csv"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func readCsv(t *testing.T, data string) [][]string {
|
|
reader := csv.NewReader(strings.NewReader(data))
|
|
var rows [][]string
|
|
for {
|
|
row, err := reader.Read()
|
|
reader.FieldsPerRecord = 0 // every row can have different number of fields
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
t.Log("row: ", row)
|
|
t.Log(err)
|
|
t.Fail()
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
return rows
|
|
}
|
|
|
|
// TestQueryResult validates construction of table columns from Query CSV result
|
|
func TestQueryResult(t *testing.T) {
|
|
const csvQueryResult = `
|
|
#group,false,false,true,true,false,false,true,true,true,true
|
|
#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string,string,string,string
|
|
#default,_result,,,,,,,,,
|
|
,result,table,_start,_stop,_time,_value,_field,_measurement,cpu,host
|
|
,,0,2020-02-25T22:17:54.068926364Z,2020-02-25T22:22:54.068926364Z,2020-02-25T22:17:57Z,0,time_steal,cpu,cpu1,rsavage.prod
|
|
,,0,2020-02-25T22:17:54.068926364Z,2020-02-25T22:22:54.068926364Z,2020-02-25T22:18:07Z,0,time_steal,cpu,cpu1,rsavage.prod
|
|
|
|
#group,false,false,true,true,false,false,true,true,true,true
|
|
#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string,string,string,string
|
|
#default,_result,,,,,,,,
|
|
,result,table,_start,_stop,_time,_value,_field,_measurement,cpu,host
|
|
,,1,2020-02-25T22:17:54.068926364Z,2020-02-25T22:22:54.068926364Z,2020-02-25T22:18:01Z,2.7263631815907954,usage_user,cpu,cpu-total,tahoecity.prod
|
|
,,1,2020-02-25T22:17:54.068926364Z,2020-02-25T22:22:54.068926364Z,2020-02-25T22:18:11Z,2.247752247752248,usage_user,cpu,cpu-total,tahoecity.prod
|
|
#unre`
|
|
|
|
var lineProtocolQueryResult = []string{
|
|
"cpu,cpu=cpu1,host=rsavage.prod time_steal=0 1582669077000000000",
|
|
"cpu,cpu=cpu1,host=rsavage.prod time_steal=0 1582669087000000000",
|
|
"cpu,cpu=cpu-total,host=tahoecity.prod usage_user=2.7263631815907954 1582669081000000000",
|
|
"cpu,cpu=cpu-total,host=tahoecity.prod usage_user=2.247752247752248 1582669091000000000",
|
|
}
|
|
|
|
table := CsvTable{}
|
|
rows := readCsv(t, csvQueryResult)
|
|
lineProtocolIndex := 0
|
|
for i, row := range rows {
|
|
rowProcessed := table.AddRow(row)
|
|
if i%6 < 4 {
|
|
require.Equal(t, rowProcessed, false, "row %d", i)
|
|
} else {
|
|
require.Equal(t, rowProcessed, true, "row %d", i)
|
|
line, _ := table.CreateLine(row)
|
|
require.Equal(t, lineProtocolQueryResult[lineProtocolIndex], line)
|
|
lineProtocolIndex++
|
|
if i%6 == 4 {
|
|
// verify table
|
|
require.GreaterOrEqual(t, len(table.columns), 10)
|
|
require.Equal(t, table.columns, table.Columns())
|
|
for j, col := range table.columns {
|
|
if j > 0 {
|
|
require.Equal(t, col.Index, j)
|
|
require.Equal(t, col.Label, rows[i-1][j])
|
|
if len(rows[i-2]) > j {
|
|
require.Equal(t, col.DefaultValue, rows[i-2][j])
|
|
} else {
|
|
// some traling data are missing
|
|
require.Equal(t, col.DefaultValue, "")
|
|
}
|
|
require.Equal(t, col.DataType, rows[i-3][j], "row %d, col %d", i-3, j)
|
|
}
|
|
}
|
|
// verify cached values
|
|
table.computeIndexes()
|
|
require.Equal(t, table.Column("_measurement"), table.cachedMeasurement)
|
|
require.Nil(t, table.Column("_no"))
|
|
require.NotNil(t, table.cachedMeasurement)
|
|
require.NotNil(t, table.cachedFieldName)
|
|
require.NotNil(t, table.cachedFieldValue)
|
|
require.NotNil(t, table.cachedTime)
|
|
require.NotNil(t, table.cachedTags)
|
|
require.Equal(t, table.Measurement().Label, "_measurement")
|
|
require.Equal(t, table.FieldName().Label, "_field")
|
|
require.Equal(t, table.FieldValue().Label, "_value")
|
|
require.Equal(t, table.Time().Label, "_time")
|
|
require.Equal(t, len(table.Tags()), 2)
|
|
require.Equal(t, table.Tags()[0].Label, "cpu")
|
|
require.Equal(t, table.Tags()[1].Label, "host")
|
|
require.Equal(t, len(table.Fields()), 0)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//Test_ignoreLeadingComment
|
|
func Test_ignoreLeadingComment(t *testing.T) {
|
|
var tests = []struct {
|
|
value string
|
|
expect string
|
|
}{
|
|
{"", ""},
|
|
{"a", "a"},
|
|
{" #whatever", " #whatever"},
|
|
{"#whatever", ""},
|
|
{"#whatever ", ""},
|
|
{"#whatever a b ", "a b "},
|
|
{"#whatever a b ", "a b "},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.value, func(t *testing.T) {
|
|
require.Equal(t, test.expect, ignoreLeadingComment(test.value))
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
// TestCsvData checks data that are writen in an annotated CSV file
|
|
func TestCsvData(t *testing.T) {
|
|
var tests = []struct {
|
|
name string
|
|
csv string
|
|
line string
|
|
}{
|
|
{
|
|
"simple1",
|
|
"_measurement,a,b\ncpu,1,1",
|
|
"cpu a=1,b=1",
|
|
},
|
|
{
|
|
"simple1b",
|
|
"_measurement,,a,b\ncpu,whatever,1,1",
|
|
"cpu a=1,b=1",
|
|
},
|
|
{
|
|
"simple2",
|
|
"_measurement\ncpu,1,1",
|
|
"", // no fields present
|
|
},
|
|
{
|
|
"simple3",
|
|
"_time\n1,1",
|
|
"", // no measurement present
|
|
},
|
|
{
|
|
"annotated1",
|
|
"#datatype measurement,,\nmeasurement,a,b\ncpu,1,2",
|
|
"cpu a=1,b=2",
|
|
},
|
|
{
|
|
"annotated2",
|
|
"#datatype measurement,tag,field\nmeasurement,a,b\ncpu,1,2",
|
|
"cpu,a=1 b=2",
|
|
},
|
|
{
|
|
"annotated3",
|
|
"#datatype measurement,tag,dateTime,field\nmeasurement,a,b,time\ncpu,1,2,3",
|
|
"cpu,a=1 time=3 2",
|
|
},
|
|
{
|
|
"annotated3_detectedTime1",
|
|
"#datatype measurement,tag,dateTime,field\nmeasurement,a,b,time\ncpu,1,2020-01-10T10:10:10Z,3",
|
|
"cpu,a=1 time=3 1578651010000000000",
|
|
},
|
|
{
|
|
"annotated3_detectedTime2",
|
|
"#datatype measurement,tag,dateTime,field\nmeasurement,a,b,time\ncpu,1,2020-01-10T10:10:10.0Z,3",
|
|
"cpu,a=1 time=3 1578651010000000000",
|
|
},
|
|
{
|
|
"annotated4",
|
|
"#datatype measurement,tag,ignore,field\nmeasurement,a,b,time\ncpu,1,2,3",
|
|
"cpu,a=1 time=3",
|
|
},
|
|
{
|
|
"annotated5",
|
|
"#datatype measurement,tag,ignore,field\nmeasurement,a,b,time\ncpu,1,2,3",
|
|
"cpu,a=1 time=3",
|
|
},
|
|
{
|
|
"annotated6",
|
|
"#datatype measurement,tag,ignore,field\n" +
|
|
"#datatypea tag,tag,\n" + // this must be ignored since it not a supported annotation
|
|
"measurement,a,b,time\ncpu,1,2,3",
|
|
"cpu,a=1 time=3",
|
|
},
|
|
{
|
|
"annotated7",
|
|
"#datatype measurement,dateTime,\nmeasurement,a,b\ncpu,2020-01-10T10:10:10.0Z,2",
|
|
"cpu b=2 1578651010000000000",
|
|
},
|
|
{
|
|
"annotated8",
|
|
"#datatype measurement,,,field\nmeasurement,_field,_value,other\ncpu,a,1,2",
|
|
"cpu a=1,other=2",
|
|
},
|
|
{
|
|
"annotated9_sortedTags",
|
|
"#datatype measurement,tag,tag,time,field\nmeasurement,b,a,c,time\ncpu,1,2,3,4",
|
|
"cpu,a=2,b=1 time=4 3",
|
|
},
|
|
{
|
|
"allFieldTypes",
|
|
"#datatype measurement,string,double,boolean,long,unsignedLong,duration,base64Binary,dateTime\n" +
|
|
"m,s,d,b,l,ul,dur,by,d1,d2,time\n" +
|
|
`cpu,"str",1.0,true,1,1,1ms,YWFh,1`,
|
|
"cpu s=\"str\",d=1,b=true,l=1i,ul=1u,dur=1000000i,by=YWFh 1",
|
|
},
|
|
{
|
|
"allFieldTypes",
|
|
"#datatype measurement,string,double,boolean,long,unsignedLong,duration,base64Binary,dateTime\n" +
|
|
"m,s,d,b,l,ul,dur,by,d1,d2,time\n" +
|
|
`cpu,"str",1.0,true,1,1,1ms,YWFh,1`,
|
|
"cpu s=\"str\",d=1,b=true,l=1i,ul=1u,dur=1000000i,by=YWFh 1",
|
|
},
|
|
{
|
|
"allFieldTypes_ignoreAdditionalDateTimes",
|
|
"#datatype ,string,double,boolean,long,unsignedLong,duration,base64Binary,dateTime:RFC3339,dateTime:RFC3339Nano,\n" +
|
|
"_measurement,s,d,b,l,ul,dur,by,d1,d2,_time\n" +
|
|
`cpu,"str",1.0,true,1,1,1ms,YWFh,2020-01-10T10:10:10Z,2020-01-10T10:10:10Z,1`,
|
|
"cpu s=\"str\",d=1,b=true,l=1i,ul=1u,dur=1000000i,by=YWFh 1",
|
|
},
|
|
{
|
|
"allExtraDataTypes",
|
|
"#datatype measurement,tag,field,ignored,dateTime\n" +
|
|
"m,t,f,i,dt\n" +
|
|
`cpu,myTag,0,myIgnored,1`,
|
|
"cpu,t=myTag f=0 1",
|
|
},
|
|
{
|
|
"allTypes_escaped",
|
|
"#datatype ,string,string,,,,\n" +
|
|
`_measurement,s1,s2,"a,","b ",c=` + "\n" +
|
|
`"cpu, ","""",\,a,b,c`,
|
|
`cpu\,\ s1="\"",s2="\\",a\,=a,b\ =b,c\==c`,
|
|
},
|
|
{
|
|
"default_values",
|
|
"#default cpu,yes,0,1\n#datatype ,tag,,\n_measurement,test,col1,_time\n,,,",
|
|
"cpu,test=yes col1=0 1",
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
rows := readCsv(t, test.csv)
|
|
table := CsvTable{}
|
|
var lines []string
|
|
for _, row := range rows {
|
|
rowProcessed := table.AddRow(row)
|
|
if rowProcessed {
|
|
line, err := table.CreateLine(row)
|
|
if err != nil && test.line != "" {
|
|
require.Nil(t, err.Error())
|
|
}
|
|
lines = append(lines, line)
|
|
}
|
|
}
|
|
require.Equal(t, []string{test.line}, lines)
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestCsvData_dataErrors validates table data errors
|
|
func TestCsvData_dataErrors(t *testing.T) {
|
|
var tests = []struct {
|
|
name string
|
|
csv string
|
|
}{
|
|
{
|
|
"error_1_is_not_dateTime:RFC3339",
|
|
"#datatype measurement,,\n#datatype ,dateTime:RFC3339,\nmeasurement,a,b\ncpu,1,2",
|
|
},
|
|
{
|
|
"error_a_fieldValue_is_not_long",
|
|
"#datatype measurement,,\n#datatype ,long,\nmeasurement,_value,_field\ncpu,a,count",
|
|
},
|
|
{
|
|
"error_a_is_not_long",
|
|
"#datatype measurement,,\n#datatype ,long,\nmeasurement,a,b\ncpu,a,2",
|
|
},
|
|
{
|
|
"error_time_is_not_time",
|
|
"#datatype measurement,tag,time,field\nmeasurement,a,b,time\ncpu,1,2020-10,3",
|
|
},
|
|
{
|
|
"error_no_measurement",
|
|
"#datatype ,\ncol1,col2\n1,2",
|
|
},
|
|
{
|
|
"error_unsupportedFieldDataType",
|
|
"#datatype ,whatever\n_measurement,col2\na,2",
|
|
},
|
|
{
|
|
"error_unsupportedFieldValueDataType",
|
|
"#datatype ,,whatever\n_measurement,_field,_value\na,1,2",
|
|
},
|
|
{
|
|
"error_no_measurement_data",
|
|
"_measurement,col1\n,2",
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
rows := readCsv(t, test.csv)
|
|
table := CsvTable{}
|
|
var errors []error
|
|
for _, row := range rows {
|
|
rowProcessed := table.AddRow(row)
|
|
if rowProcessed {
|
|
_, err := table.CreateLine(row)
|
|
if err != nil {
|
|
errors = append(errors, err)
|
|
}
|
|
}
|
|
}
|
|
require.Equal(t, 1, len(errors))
|
|
// fmt.Println(errors[0])
|
|
require.NotNil(t, errors[0].Error())
|
|
// LineLabel is the same as Label in all test columns
|
|
for _, col := range table.Columns() {
|
|
require.Equal(t, col.Label, col.LineLabel())
|
|
}
|
|
})
|
|
}
|
|
}
|