influxdb/pkg/csv2lp/csv2lp_test.go

package csv2lp

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"
)

// Test_CsvToLineProtocol tests conversion of annotated CSV data to line protocol data
func Test_CsvToLineProtocol(t *testing.T) {
	var tests = []struct {
		name  string
		csv   string
		lines string
		err   string
	}{
		{
			"simple1",
			"_measurement,a,b\ncpu,1,1\ncpu,b2\n",
			"cpu a=1,b=1\ncpu a=b2\n",
			"",
		},
		{
			"simple1_withSep",
			"sep=;\n_measurement;a;b\ncpu;1;1\ncpu;b2\n",
			"cpu a=1,b=1\ncpu a=b2\n",
			"",
		},
		{
			"simple2",
			"_measurement,a,b\ncpu,1,1\ncpu,\n",
			"",
			"no field data",
		},
		{
			"simple3",
			"_measurement,a,_time\ncpu,1,1\ncpu,2,invalidTime\n",
			"",
			"_time", // error in _time column
		},
		{
			"constant_annotations",
			"#constant,measurement,,cpu\n" +
				"#constant,tag,xpu,xpu1\n" +
				"#constant,tag,cpu,cpu1\n" +
				"#constant,long,of,100\n" +
				"#constant,dateTime,,2\n" +
				"x,y\n" +
				"1,2\n" +
				"3,4\n",
			"cpu,cpu=cpu1,xpu=xpu1 x=1,y=2,of=100i 2\n" +
				"cpu,cpu=cpu1,xpu=xpu1 x=3,y=4,of=100i 2\n",
			"", // no error
		},
		{
			"timezone_annotation-0100",
			"#timezone,-0100\n" +
				"#constant,measurement,cpu\n" +
				"#constant,dateTime:2006-01-02,1970-01-01\n" +
				"x,y\n" +
				"1,2\n",
			"cpu x=1,y=2 3600000000000\n",
			"", // no error
		},
		{
			"timezone_annotation_EST",
			"#timezone,EST\n" +
				"#constant,measurement,cpu\n" +
				"#constant,dateTime:2006-01-02,1970-01-01\n" +
				"x,y\n" +
				"1,2\n",
			"cpu x=1,y=2 18000000000000\n",
			"", // no error
		},
	}
	bufferSizes := []int{40, 7, 3, 1}

	for _, test := range tests {
		for _, bufferSize := range bufferSizes {
			t.Run(test.name+"_"+strconv.Itoa(bufferSize), func(t *testing.T) {
				reader := CsvToLineProtocol(strings.NewReader(test.csv))
				buffer := make([]byte, bufferSize)
				lines := make([]byte, 0, 100)
				for {
					n, err := reader.Read(buffer)
					if err != nil {
						if err == io.EOF {
							break
						}
						if test.err != "" {
							// fmt.Println(err)
							if err := err.Error(); !strings.Contains(err, test.err) {
								require.Equal(t, err, test.err)
							}
							return
						}
						require.Nil(t, err.Error())
						break
					}
					lines = append(lines, buffer[:n]...)
				}
				if test.err == "" {
					require.Equal(t, test.lines, string(lines))
				} else {
					require.Fail(t, "error message with '"+test.err+"' expected")
				}
			})
		}
	}
}

// Test_CsvToLineProtocol_LogTableColumns checks correct logging of table columns
func Test_CsvToLineProtocol_LogTableColumns(t *testing.T) {
	var buf bytes.Buffer
	log.SetOutput(&buf)
	oldFlags := log.Flags()
	log.SetFlags(0)
	oldPrefix := log.Prefix()
	prefix := "::PREFIX::"
	log.SetPrefix(prefix)
	defer func() {
		log.SetOutput(os.Stderr)
		log.SetFlags(oldFlags)
		log.SetPrefix(oldPrefix)
	}()

	csv := "_measurement,a,b\ncpu,1,1\ncpu,b2\n"

	reader := CsvToLineProtocol(strings.NewReader(csv)).LogTableColumns(true)
	require.False(t, reader.skipRowOnError)
	require.True(t, reader.logTableDataColumns)
	// read all the data
	ioutil.ReadAll(reader)

	out := buf.String()
	// fmt.Println(out)
	messages := strings.Count(out, prefix)
	require.Equal(t, messages, 1)
}

// Test_CsvToLineProtocol_LogTimeZoneWarning checks correct logging of timezone warning
func Test_CsvToLineProtocol_LogTimeZoneWarning(t *testing.T) {
	var buf bytes.Buffer
	log.SetOutput(&buf)
	oldFlags := log.Flags()
	log.SetFlags(0)
	oldPrefix := log.Prefix()
	prefix := "::PREFIX::"
	log.SetPrefix(prefix)
	defer func() {
		log.SetOutput(os.Stderr)
		log.SetFlags(oldFlags)
		log.SetPrefix(oldPrefix)
	}()

	csv := "#timezone 1\n" +
		"#constant,dateTime:2006-01-02,1970-01-01\n" +
		"_measurement,a,b\ncpu,1,1"

	reader := CsvToLineProtocol(strings.NewReader(csv))
	bytes, _ := ioutil.ReadAll(reader)

	out := buf.String()
	// fmt.Println(out) // "::PREFIX::WARNING:  #timezone annotation: unknown time zone 1
	messages := strings.Count(out, prefix)
	require.Equal(t, messages, 1)
	require.Equal(t, string(bytes), "cpu a=1,b=1 0\n")
}

// Test_CsvToLineProtocol_SkipRowOnError tests that error rows are skipped
func Test_CsvToLineProtocol_SkipRowOnError(t *testing.T) {
	var buf bytes.Buffer
	log.SetOutput(&buf)
	oldFlags := log.Flags()
	log.SetFlags(0)
	oldPrefix := log.Prefix()
	prefix := "::PREFIX::"
	log.SetPrefix(prefix)
	defer func() {
		log.SetOutput(os.Stderr)
		log.SetFlags(oldFlags)
		log.SetPrefix(oldPrefix)
	}()

	csv := "_measurement,a,_time\n,1,1\ncpu,2,2\ncpu,3,3a\n"

	reader := CsvToLineProtocol(strings.NewReader(csv)).SkipRowOnError(true)
	require.Equal(t, reader.skipRowOnError, true)
	require.Equal(t, reader.logTableDataColumns, false)
	// read all the data
	ioutil.ReadAll(reader)

	out := buf.String()
	// fmt.Println(out)
	messages := strings.Count(out, prefix)
	require.Equal(t, messages, 2)
}

// Test_CsvLineError tests CsvLineError error format
func Test_CsvLineError(t *testing.T) {
	var tests = []struct {
		err   CsvLineError
		value string
	}{
		{
			CsvLineError{Line: 1, Err: errors.New("cause")},
			"line 1: cause",
		},
		{
			CsvLineError{Line: 2, Err: CsvColumnError{"a", errors.New("cause")}},
			"line 2: column 'a': cause",
		},
	}
	for _, test := range tests {
		require.Equal(t, test.value, test.err.Error())
	}
}