package wal import ( "bytes" "fmt" "io/ioutil" "math/rand" "os" "reflect" "testing" "time" "github.com/influxdb/influxdb/influxql" "github.com/influxdb/influxdb/models" "github.com/influxdb/influxdb/tsdb" ) func TestWAL_WritePoints(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) // Test that we can write to two different series p1 := parsePoint("cpu,host=A value=23.2 1", codec) p2 := parsePoint("cpu,host=A value=25.3 4", codec) p3 := parsePoint("cpu,host=B value=1.0 1", codec) if err := log.WritePoints([]models.Point{p1, p2, p3}, nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } c := log.Cursor("cpu,host=A", []string{"value"}, codec, true) k, v := c.SeekTo(1) // ensure the series are there and points are in order if v.(float64) != 23.2 { t.Fatalf("expected to seek to first point but got key and value: %v %v", k, v) } k, v = c.Next() if v.(float64) != 25.3 { t.Fatalf("expected to seek to first point but got key and value: %v %v", k, v) } k, v = c.Next() if k != tsdb.EOF { t.Fatalf("expected nil on last seek: %v %v", k, v) } c = log.Cursor("cpu,host=B", []string{"value"}, codec, true) k, v = c.Next() if v.(float64) != 1.0 { t.Fatalf("expected to seek to first point but got key and value: %v %v", k, v) } // ensure that we can close and re-open the log with points getting to the index log.Close() points := make([]map[string][][]byte, 0) log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { points = append(points, pointsByKey) return nil }} if err := log.Open(); err != nil { t.Fatal("error opening log", err) } p := points[0] if len(p["cpu,host=A"]) != 2 { t.Fatal("expected two points for cpu,host=A flushed to index") } if len(p["cpu,host=B"]) != 1 { t.Fatal("expected one point for cpu,host=B flushed to index") } // ensure we can write new points into the series p4 := parsePoint("cpu,host=A value=1.0 7", codec) // ensure we can write an all new series p5 := parsePoint("cpu,host=C value=1.4 2", codec) // ensure we can write a point out of order and get it back p6 := parsePoint("cpu,host=A value=1.3 2", codec) // // ensure we can write to a new partition // p7 := parsePoint("cpu,region=west value=2.2", codec) if err := log.WritePoints([]models.Point{p4, p5, p6}, nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } c = log.Cursor("cpu,host=A", []string{"value"}, codec, true) if _, v := c.Next(); v.(float64) != 1.3 { t.Fatal("order wrong, expected p6") } if _, v := c.Next(); v.(float64) != 1.0 { t.Fatal("order wrong, expected p6") } c = log.Cursor("cpu,host=C", []string{"value"}, codec, true) if _, v := c.Next(); v.(float64) != 1.4 { t.Fatal("order wrong, expected p6") } if err := log.Close(); err != nil { t.Fatal("error closing log", err) } points = make([]map[string][][]byte, 0) if err := log.Open(); err != nil { t.Fatal("error opening log", err) } p = points[0] if len(p["cpu,host=A"]) != 2 { t.Fatal("expected two points for cpu,host=A flushed to index") } if len(p["cpu,host=C"]) != 1 { t.Fatal("expected one point for cpu,host=B flushed to index") } } func TestWAL_CorruptDataLengthSize(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) // test that we can write to two different series p1 := parsePoint("cpu,host=A value=23.2 1", codec) p2 := parsePoint("cpu,host=A value=25.3 4", codec) if err := log.WritePoints([]models.Point{p1, p2}, nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } c := log.Cursor("cpu,host=A", []string{"value"}, codec, true) if _, v := c.Next(); v.(float64) != 23.2 { t.Fatal("p1 value wrong") } if _, v := c.Next(); v.(float64) != 25.3 { t.Fatal("p2 value wrong") } if _, v := c.Next(); v != nil { t.Fatal("expected cursor to return nil") } // now write junk data and ensure that we can close, re-open and read f := log.partition.currentSegmentFile f.Write([]byte{0x23, 0x12}) f.Sync() log.Close() points := make([]map[string][][]byte, 0) log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { points = append(points, pointsByKey) return nil }} log.Open() if p := points[0]; len(p["cpu,host=A"]) != 2 { t.Fatal("expected two points for cpu,host=A") } // now write new data and ensure it's all good p3 := parsePoint("cpu,host=A value=29.2 6", codec) if err := log.WritePoints([]models.Point{p3}, nil, nil); err != nil { t.Fatalf("failed to write point: %s", err.Error()) } c = log.Cursor("cpu,host=A", []string{"value"}, codec, true) if _, v := c.Next(); v.(float64) != 29.2 { t.Fatal("p3 value wrong") } log.Close() points = make([]map[string][][]byte, 0) log.Open() if p := points[0]; len(p["cpu,host=A"]) != 1 { t.Fatal("expected two points for cpu,host=A") } } func TestWAL_CorruptDataBlock(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) // test that we can write to two different series p1 := parsePoint("cpu,host=A value=23.2 1", codec) p2 := parsePoint("cpu,host=A value=25.3 4", codec) if err := log.WritePoints([]models.Point{p1, p2}, nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } c := log.Cursor("cpu,host=A", []string{"value"}, codec, true) if _, v := c.Next(); v.(float64) != 23.2 { t.Fatal("p1 value wrong") } if _, v := c.Next(); v.(float64) != 25.3 { t.Fatal("p2 value wrong") } if _, v := c.Next(); v != nil { t.Fatal("expected cursor to return nil") } // now write junk data and ensure that we can close, re-open and read f := log.partition.currentSegmentFile f.Write(u64tob(23)) // now write a bunch of garbage for i := 0; i < 1000; i++ { f.Write([]byte{0x23, 0x78, 0x11, 0x33}) } f.Sync() log.Close() points := make([]map[string][][]byte, 0) log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { points = append(points, pointsByKey) return nil }} log.Open() if p := points[0]; len(p["cpu,host=A"]) != 2 { t.Fatal("expected two points for cpu,host=A") } // now write new data and ensure it's all good p3 := parsePoint("cpu,host=A value=29.2 6", codec) if err := log.WritePoints([]models.Point{p3}, nil, nil); err != nil { t.Fatalf("failed to write point: %s", err.Error()) } c = log.Cursor("cpu,host=A", []string{"value"}, codec, true) if _, v := c.Next(); v.(float64) != 29.2 { t.Fatal("p3 value wrong", p3.Data(), v) } log.Close() points = make([]map[string][][]byte, 0) log.Open() if p := points[0]; len(p["cpu,host=A"]) != 1 { t.Fatal("expected two points for cpu,host=A") } } // Ensure the wal forces a full flush after not having a write in a given interval of time func TestWAL_CompactAfterTimeWithoutWrite(t *testing.T) { log := openTestWAL() // set this low log.flushCheckInterval = 10 * time.Millisecond log.FlushColdInterval = 500 * time.Millisecond defer log.Close() defer os.RemoveAll(log.path) points := make([]map[string][][]byte, 0) log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { points = append(points, pointsByKey) return nil }} if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) numSeries := 100 b := make([]byte, 70*5000) for i := 1; i <= 10; i++ { buf := bytes.NewBuffer(b) for j := 1; j <= numSeries; j++ { buf.WriteString(fmt.Sprintf("cpu,host=A,region=uswest%d value=%.3f %d\n", j, rand.Float64(), i)) } // write the batch out if err := log.WritePoints(parsePoints(buf.String(), codec), nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } buf = bytes.NewBuffer(b) } // ensure we have some data c := log.Cursor("cpu,host=A,region=uswest10", []string{"value"}, codec, true) k, _ := c.Next() if k != 1 { t.Fatalf("expected first data point but got one with key: %v", k) } time.Sleep(700 * time.Millisecond) // ensure that as a whole its not ready for flushing yet if f := log.partition.shouldFlush(); f != noFlush { t.Fatalf("expected partition 1 to return noFlush from shouldFlush %v", f) } // ensure that the partition is empty if log.partition.memorySize != 0 || len(log.partition.cache) != 0 { t.Fatal("expected partition to be empty") } // ensure that we didn't bother to open a new segment file if log.partition.currentSegmentFile != nil { t.Fatal("expected partition to not have an open segment file") } } func TestWAL_SeriesAndFieldsGetPersisted(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) var measurementsToIndex map[string]*tsdb.MeasurementFields var seriesToIndex []*tsdb.SeriesCreate log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { measurementsToIndex = measurementFieldsToSave seriesToIndex = append(seriesToIndex, seriesToCreate...) return nil }} // test that we can write to two different series p1 := parsePoint("cpu,host=A value=23.2 1", codec) p2 := parsePoint("cpu,host=A value=25.3 4", codec) p3 := parsePoint("cpu,host=B value=1.0 1", codec) seriesToCreate := []*tsdb.SeriesCreate{ {Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "A"})), map[string]string{"host": "A"})}, {Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "B"})), map[string]string{"host": "B"})}, } measaurementsToCreate := map[string]*tsdb.MeasurementFields{ "cpu": { Fields: map[string]*tsdb.Field{ "value": {ID: 1, Name: "value"}, }, }, } if err := log.WritePoints([]models.Point{p1, p2, p3}, measaurementsToCreate, seriesToCreate); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } // now close it and see if loading the metadata index will populate the measurement and series info log.Close() idx := tsdb.NewDatabaseIndex() mf := make(map[string]*tsdb.MeasurementFields) if err := log.LoadMetadataIndex(idx, mf); err != nil { t.Fatalf("error loading metadata index: %s", err.Error()) } s := idx.Series("cpu,host=A") if s == nil { t.Fatal("expected to find series cpu,host=A in index") } s = idx.Series("cpu,host=B") if s == nil { t.Fatal("expected to find series cpu,host=B in index") } m := mf["cpu"] if m == nil { t.Fatal("expected to find measurement fields for cpu", mf) } if m.Fields["value"] == nil { t.Fatal("expected to find field definition for 'value'") } // ensure that they were actually flushed to the index. do it this way because the annoying deepequal doessn't really work for these for i, s := range seriesToCreate { if seriesToIndex[i].Measurement != s.Measurement { t.Fatal("expected measurement to be the same") } if seriesToIndex[i].Series.Key != s.Series.Key { t.Fatal("expected series key to be the same") } if !reflect.DeepEqual(seriesToIndex[i].Series.Tags, s.Series.Tags) { t.Fatal("expected series tags to be the same") } } // ensure that the measurement fields were flushed to the index for k, v := range measaurementsToCreate { m := measurementsToIndex[k] if m == nil { t.Fatalf("measurement %s wasn't indexed", k) } if !reflect.DeepEqual(m.Fields, v.Fields) { t.Fatal("measurement fields not equal") } } // now open and close the log and try to reload the metadata index, which should now be empty if err := log.Open(); err != nil { t.Fatalf("error opening log: %s", err.Error()) } if err := log.Close(); err != nil { t.Fatalf("error closing log: %s", err.Error()) } idx = tsdb.NewDatabaseIndex() mf = make(map[string]*tsdb.MeasurementFields) if err := log.LoadMetadataIndex(idx, mf); err != nil { t.Fatalf("error loading metadata index: %s", err.Error()) } if len(idx.Measurements()) != 0 || len(mf) != 0 { t.Fatal("expected index and measurement fields to be empty") } } func TestWAL_DeleteSeries(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) var seriesToIndex []*tsdb.SeriesCreate var points map[string][][]byte log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { points = pointsByKey seriesToIndex = append(seriesToIndex, seriesToCreate...) return nil }} seriesToCreate := []*tsdb.SeriesCreate{ {Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "A"})), map[string]string{"host": "A"})}, {Series: tsdb.NewSeries(string(models.MakeKey([]byte("cpu"), map[string]string{"host": "B"})), map[string]string{"host": "B"})}, } if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } // test that we can write to two different series p1 := parsePoint("cpu,host=A value=23.2 1", codec) p2 := parsePoint("cpu,host=B value=0.9 2", codec) p3 := parsePoint("cpu,host=A value=25.3 4", codec) p4 := parsePoint("cpu,host=B value=1.0 3", codec) if err := log.WritePoints([]models.Point{p1, p2, p3, p4}, nil, seriesToCreate); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } // ensure data is there c := log.Cursor("cpu,host=A", []string{"value"}, codec, true) if k, _ := c.Next(); k != 1 { t.Fatal("expected data point for cpu,host=A") } c = log.Cursor("cpu,host=B", []string{"value"}, codec, true) if k, _ := c.Next(); k != 2 { t.Fatal("expected data point for cpu,host=B") } // delete the series and ensure metadata was flushed and data is gone if err := log.DeleteSeries([]string{"cpu,host=B"}); err != nil { t.Fatalf("error deleting series: %s", err.Error()) } // ensure data is there if len(points["cpu,host=A"]) != 2 { t.Fatal("expected cpu,host=A to be flushed to the index") } if len(points["cpu,host=B"]) != 0 { t.Fatal("expected cpu,host=B to have no points in index") } c = log.Cursor("cpu,host=A", []string{"value"}, codec, true) if k, _ := c.Next(); k != tsdb.EOF { t.Fatal("expected data to be out of the cache cpu,host=A") } // ensure series is deleted c = log.Cursor("cpu,host=B", []string{"value"}, codec, true) if k, _ := c.Next(); k != tsdb.EOF { t.Fatal("expected no data for cpu,host=B") } // ensure that they were actually flushed to the index. do it this way because the annoying deepequal doessn't really work for these for i, s := range seriesToCreate { if seriesToIndex[i].Measurement != s.Measurement { t.Fatal("expected measurement to be the same") } if seriesToIndex[i].Series.Key != s.Series.Key { t.Fatal("expected series key to be the same") } if !reflect.DeepEqual(seriesToIndex[i].Series.Tags, s.Series.Tags) { t.Fatal("expected series tags to be the same") } } // close and re-open the WAL to ensure that the data didn't show back up if err := log.Close(); err != nil { t.Fatalf("error closing log: %s", err.Error()) } points = make(map[string][][]byte) if err := log.Open(); err != nil { t.Fatalf("error opening log: %s", err.Error()) } // ensure data wasn't flushed on open if len(points) != 0 { t.Fatal("expected no data to be flushed on open") } } func TestWAL_QueryDuringCompaction(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) var points []map[string][][]byte finishCompaction := make(chan struct{}) log.Index = &testIndexWriter{fn: func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { points = append(points, pointsByKey) finishCompaction <- struct{}{} return nil }} if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) // test that we can write to two different series p1 := parsePoint("cpu,host=A value=23.2 1", codec) if err := log.WritePoints([]models.Point{p1}, nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } verify := func() { c := log.Cursor("cpu,host=A", []string{"value"}, codec, true) k, v := c.SeekTo(1) // ensure the series are there and points are in order if v.(float64) != 23.2 { <-finishCompaction t.Fatalf("expected to seek to first point but got key and value: %v %v", k, v) } } verify() go func() { log.Flush() }() time.Sleep(100 * time.Millisecond) verify() <-finishCompaction verify() } func TestWAL_PointsSorted(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) // test that we can write to two different series p1 := parsePoint("cpu,host=A value=1.1 1", codec) p2 := parsePoint("cpu,host=A value=4.4 4", codec) p3 := parsePoint("cpu,host=A value=2.2 2", codec) p4 := parsePoint("cpu,host=A value=6.6 6", codec) if err := log.WritePoints([]models.Point{p1, p2, p3, p4}, nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } c := log.Cursor("cpu,host=A", []string{"value"}, codec, true) if k, _ := c.Next(); k != 1 { t.Fatal("points out of order") } if k, _ := c.Next(); k != 2 { t.Fatal("points out of order") } if k, _ := c.Next(); k != 4 { t.Fatal("points out of order") } if k, _ := c.Next(); k != 6 { t.Fatal("points out of order") } } func TestWAL_Cursor_Reverse(t *testing.T) { log := openTestWAL() defer log.Close() defer os.RemoveAll(log.path) if err := log.Open(); err != nil { t.Fatalf("couldn't open wal: %s", err.Error()) } codec := tsdb.NewFieldCodec(map[string]*tsdb.Field{ "value": { ID: uint8(1), Name: "value", Type: influxql.Float, }, }) // test that we can write to two different series p1 := parsePoint("cpu,host=A value=1.1 1", codec) p2 := parsePoint("cpu,host=A value=4.4 4", codec) p3 := parsePoint("cpu,host=A value=2.2 2", codec) p4 := parsePoint("cpu,host=A value=6.6 6", codec) if err := log.WritePoints([]models.Point{p1, p2, p3, p4}, nil, nil); err != nil { t.Fatalf("failed to write points: %s", err.Error()) } c := log.Cursor("cpu,host=A", []string{"value"}, codec, false) k, _ := c.Next() if k != 6 { t.Fatal("points out of order") } if k, _ := c.Next(); k != 4 { t.Fatal("points out of order") } if k, _ := c.Next(); k != 2 { t.Fatal("points out of order") } if k, _ := c.Next(); k != 1 { t.Fatal("points out of order") } } type testIndexWriter struct { fn func(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error } func (t *testIndexWriter) WriteIndex(pointsByKey map[string][][]byte, measurementFieldsToSave map[string]*tsdb.MeasurementFields, seriesToCreate []*tsdb.SeriesCreate) error { return t.fn(pointsByKey, measurementFieldsToSave, seriesToCreate) } func openTestWAL() *Log { dir, err := ioutil.TempDir("", "wal-test") if err != nil { panic("couldn't get temp dir") } return NewLog(dir) } func parsePoints(buf string, codec *tsdb.FieldCodec) []models.Point { points, err := models.ParsePointsString(buf) if err != nil { panic(fmt.Sprintf("couldn't parse points: %s", err.Error())) } for _, p := range points { b, err := codec.EncodeFields(p.Fields()) if err != nil { panic(fmt.Sprintf("couldn't encode fields: %s", err.Error())) } p.SetData(b) } return points } func parsePoint(buf string, codec *tsdb.FieldCodec) models.Point { return parsePoints(buf, codec)[0] }