diff --git a/CHANGELOG.md b/CHANGELOG.md index 8596bc2fac..7610fd50c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ - [#4357](https://github.com/influxdb/influxdb/issues/4357): Fix similar float values encoding overflow Thanks @dgryski! - [#4344](https://github.com/influxdb/influxdb/issues/4344): Make client.Write default to client.precision if none is given. - [#3429](https://github.com/influxdb/influxdb/issues/3429)): Incorrect parsing of regex containing '/' +- [#4374](https://github.com/influxdb/influxdb/issues/4374)): Add tsm1 quickcheck tests ## v0.9.4 [2015-09-14] diff --git a/tsdb/engine/tsm1/bool_test.go b/tsdb/engine/tsm1/bool_test.go index ed68987afd..0530f8d13d 100644 --- a/tsdb/engine/tsm1/bool_test.go +++ b/tsdb/engine/tsm1/bool_test.go @@ -1,7 +1,9 @@ package tsm1_test import ( + "reflect" "testing" + "testing/quick" "github.com/influxdb/influxdb/tsdb/engine/tsm1" ) @@ -71,3 +73,33 @@ func Test_BoolEncoder_Multi_Compressed(t *testing.T) { t.Fatalf("unexpected next value: got true, exp false") } } + +func Test_BoolEncoder_Quick(t *testing.T) { + quick.Check(func(values []bool) bool { + // Write values to encoder. + enc := tsm1.NewBoolEncoder() + for _, v := range values { + enc.Write(v) + } + + // Retrieve compressed bytes. + buf, err := enc.Bytes() + if err != nil { + t.Fatal(err) + } + + // Read values out of decoder. + got := make([]bool, 0, len(values)) + dec := tsm1.NewBoolDecoder(buf) + for dec.Next() { + got = append(got, dec.Read()) + } + + // Verify that input and output values match. + if !reflect.DeepEqual(values, got) { + t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got) + } + + return true + }, nil) +} diff --git a/tsdb/engine/tsm1/cursor_test.go b/tsdb/engine/tsm1/cursor_test.go new file mode 100644 index 0000000000..409a26da7d --- /dev/null +++ b/tsdb/engine/tsm1/cursor_test.go @@ -0,0 +1,175 @@ +package tsm1_test + +import ( + "github.com/influxdb/influxdb/tsdb" + "github.com/influxdb/influxdb/tsdb/engine/tsm1" + "math/rand" + "reflect" + "sort" + "testing" + "testing/quick" +) + +func TestCombinedEngineCursor_Quick(t *testing.T) { + const tmin = 0 + quick.Check(func(wc, ec *Cursor, ascending bool, seek int64) bool { + c := tsm1.NewCombinedEngineCursor(wc, ec, ascending) + // Read from cursor. + got := make([]int64, 0) + for k, _ := c.SeekTo(seek); k != tsdb.EOF; k, _ = c.Next() { + got = append(got, k) + } + // Merge cursors items. + merged := MergeCursorItems(wc.items, ec.items) + if !ascending { + sort.Sort(sort.Reverse(CursorItems(merged))) + } + // Filter out items outside of seek range. + exp := make([]int64, 0) + for _, item := range merged { + if (ascending && item.Key < seek) || (!ascending && item.Key > seek) { + continue + } + exp = append(exp, item.Key) + } + if !reflect.DeepEqual(got, exp) { + t.Fatalf("mismatch: seek=%v, ascending=%v\n\ngot=%#v\n\nexp=%#v\n\n", seek, ascending, got, exp) + } + return true + }, &quick.Config{Values: func(values []reflect.Value, rand *rand.Rand) { + ascending := rand.Intn(1) == 1 + values[0] = reflect.ValueOf(GenerateCursor(tmin, 10, ascending, rand)) + values[1] = reflect.ValueOf(GenerateCursor(tmin, 10, ascending, rand)) + values[2] = reflect.ValueOf(ascending) + values[3] = reflect.ValueOf(rand.Int63n(100)) + }}) +} + +// Cursor represents a simple test cursor that implements tsdb.Cursor. +type Cursor struct { + i int + items []CursorItem + ascending bool +} + +// NewCursor returns a new instance of Cursor. +func NewCursor(items []CursorItem, ascending bool) *Cursor { + c := &Cursor{ + items: items, + ascending: ascending, + } + // Set initial position depending on cursor direction. + if ascending { + c.i = -1 + } else { + c.i = len(c.items) + } + return c +} + +// CursorItem represents an item in a test cursor. +type CursorItem struct { + Key int64 + Value interface{} +} + +// SeekTo moves the cursor to the first key greater than or equal to seek. +func (c *Cursor) SeekTo(seek int64) (key int64, value interface{}) { + if c.ascending { + for i, item := range c.items { + if item.Key >= seek { + c.i = i + return item.Key, item.Value + } + } + } else { + for i := len(c.items) - 1; i >= 0; i-- { + if item := c.items[i]; item.Key <= seek { + c.i = i + return item.Key, item.Value + } + } + } + c.i = len(c.items) + return tsdb.EOF, nil +} + +// Next returns the next key/value from the cursor. +func (c *Cursor) Next() (key int64, value interface{}) { + if c.ascending { + c.i++ + if c.i >= len(c.items) { + return tsdb.EOF, nil + } + } else if !c.ascending { + c.i-- + if c.i < 0 { + return tsdb.EOF, nil + } + } + return c.items[c.i].Key, c.items[c.i].Value +} + +// Ascending returns true if the cursor moves in ascending order. +func (c *Cursor) Ascending() bool { return c.ascending } + +// CursorItems represents a list of CursorItem objects. +type CursorItems []CursorItem + +func (a CursorItems) Len() int { return len(a) } +func (a CursorItems) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a CursorItems) Less(i, j int) bool { return a[i].Key < a[j].Key } + +// Keys returns a list of keys. +func (a CursorItems) Keys() []int64 { + keys := make([]int64, len(a)) + for i := range a { + keys[i] = a[i].Key + } + return keys +} + +// GenerateCursor generates a cursor with a random data. +func GenerateCursor(tmin, step int64, ascending bool, rand *rand.Rand) *Cursor { + key := tmin + rand.Int63n(10) + items := make([]CursorItem, 0) + for i, n := 0, rand.Intn(100); i < n; i++ { + items = append(items, CursorItem{ + Key: key, + Value: int64(0), + }) + key += rand.Int63n(10) + } + return NewCursor(items, ascending) +} + +// MergeCursorItems merges items in a & b together. +// If two items share a timestamp then a takes precendence. +func MergeCursorItems(a, b []CursorItem) []CursorItem { + items := make([]CursorItem, 0) + var ai, bi int + for { + if ai < len(a) && bi < len(b) { + if ak, bk := a[ai].Key, b[bi].Key; ak == bk { + items = append(items, a[ai]) + ai++ + bi++ + } else if ak < bk { + items = append(items, a[ai]) + ai++ + } else { + items = append(items, b[bi]) + bi++ + } + } else if ai < len(a) { + items = append(items, a[ai]) + ai++ + } else if bi < len(b) { + items = append(items, b[bi]) + bi++ + } else { + break + } + } + return items +} diff --git a/tsdb/engine/tsm1/float.go b/tsdb/engine/tsm1/float.go index 6be42352ff..2d050662c2 100644 --- a/tsdb/engine/tsm1/float.go +++ b/tsdb/engine/tsm1/float.go @@ -154,6 +154,13 @@ func (it *FloatDecoder) Next() bool { if it.first { it.first = false + + // mark as finished if there were no values. + if math.IsNaN(it.val) { + it.finished = true + return false + } + return true } diff --git a/tsdb/engine/tsm1/float_test.go b/tsdb/engine/tsm1/float_test.go index fc9ebfd358..7419558f87 100644 --- a/tsdb/engine/tsm1/float_test.go +++ b/tsdb/engine/tsm1/float_test.go @@ -1,7 +1,9 @@ package tsm1_test import ( + "reflect" "testing" + "testing/quick" "github.com/influxdb/influxdb/tsdb/engine/tsm1" ) @@ -174,6 +176,34 @@ func TestFloatEncoder_Roundtrip(t *testing.T) { } } +func Test_FloatEncoder_Quick(t *testing.T) { + quick.Check(func(values []float64) bool { + // Write values to encoder. + enc := tsm1.NewFloatEncoder() + for _, v := range values { + enc.Push(v) + } + enc.Finish() + + // Read values out of decoder. + got := make([]float64, 0, len(values)) + dec, err := tsm1.NewFloatDecoder(enc.Bytes()) + if err != nil { + t.Fatal(err) + } + for dec.Next() { + got = append(got, dec.Values()) + } + + // Verify that input and output values match. + if !reflect.DeepEqual(values, got) { + t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got) + } + + return true + }, nil) +} + func BenchmarkFloatEncoder(b *testing.B) { for i := 0; i < b.N; i++ { s := tsm1.NewFloatEncoder() diff --git a/tsdb/engine/tsm1/int.go b/tsdb/engine/tsm1/int.go index 0cd6c20c82..a3c262d3c7 100644 --- a/tsdb/engine/tsm1/int.go +++ b/tsdb/engine/tsm1/int.go @@ -2,7 +2,7 @@ package tsm1 // Int64 encoding uses two different strategies depending on the range of values in // the uncompressed data. Encoded values are first encoding used zig zag encoding. -// This interleaves postiive and negative integers across a range of positive integers. +// This interleaves positive and negative integers across a range of positive integers. // // For example, [-2,-1,0,1] becomes [3,1,0,2]. See // https://developers.google.com/protocol-buffers/docs/encoding?hl=en#signed-integers diff --git a/tsdb/engine/tsm1/int_test.go b/tsdb/engine/tsm1/int_test.go index 5f3262d820..a74d27e082 100644 --- a/tsdb/engine/tsm1/int_test.go +++ b/tsdb/engine/tsm1/int_test.go @@ -2,7 +2,9 @@ package tsm1_test import ( "math" + "reflect" "testing" + "testing/quick" "github.com/influxdb/influxdb/tsdb/engine/tsm1" ) @@ -255,6 +257,39 @@ func Test_Int64Encoder_AllNegative(t *testing.T) { } +func Test_Int64Encoder_Quick(t *testing.T) { + quick.Check(func(values []int64) bool { + // Write values to encoder. + enc := tsm1.NewInt64Encoder() + for _, v := range values { + enc.Write(v) + } + + // Retrieve encoded bytes from encoder. + buf, err := enc.Bytes() + if err != nil { + t.Fatal(err) + } + + // Read values out of decoder. + got := make([]int64, 0, len(values)) + dec := tsm1.NewInt64Decoder(buf) + for dec.Next() { + if err := dec.Error(); err != nil { + t.Fatal(err) + } + got = append(got, dec.Read()) + } + + // Verify that input and output values match. + if !reflect.DeepEqual(values, got) { + t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got) + } + + return true + }, nil) +} + func BenchmarkInt64Encoder(b *testing.B) { enc := tsm1.NewInt64Encoder() x := make([]int64, 1024) diff --git a/tsdb/engine/tsm1/string_test.go b/tsdb/engine/tsm1/string_test.go index f5143514ec..7c0e56b023 100644 --- a/tsdb/engine/tsm1/string_test.go +++ b/tsdb/engine/tsm1/string_test.go @@ -2,7 +2,9 @@ package tsm1 import ( "fmt" + "reflect" "testing" + "testing/quick" ) func Test_StringEncoder_NoValues(t *testing.T) { @@ -83,3 +85,39 @@ func Test_StringEncoder_Multi_Compressed(t *testing.T) { t.Fatalf("unexpected next value: got true, exp false") } } + +func Test_StringEncoder_Quick(t *testing.T) { + quick.Check(func(values []string) bool { + // Write values to encoder. + enc := NewStringEncoder() + for _, v := range values { + enc.Write(v) + } + + // Retrieve encoded bytes from encoder. + buf, err := enc.Bytes() + if err != nil { + t.Fatal(err) + } + + // Read values out of decoder. + got := make([]string, 0, len(values)) + dec, err := NewStringDecoder(buf) + if err != nil { + t.Fatal(err) + } + for dec.Next() { + if err := dec.Error(); err != nil { + t.Fatal(err) + } + got = append(got, dec.Read()) + } + + // Verify that input and output values match. + if !reflect.DeepEqual(values, got) { + t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", values, got) + } + + return true + }, nil) +} diff --git a/tsdb/engine/tsm1/timestamp_test.go b/tsdb/engine/tsm1/timestamp_test.go index 402a6578a1..8f4410a041 100644 --- a/tsdb/engine/tsm1/timestamp_test.go +++ b/tsdb/engine/tsm1/timestamp_test.go @@ -1,7 +1,9 @@ package tsm1 import ( + "reflect" "testing" + "testing/quick" "time" ) @@ -353,6 +355,41 @@ func Test_TimeEncoder_220SecondDelta(t *testing.T) { } } +func Test_TimeEncoder_Quick(t *testing.T) { + quick.Check(func(values []int64) bool { + // Write values to encoder. + enc := NewTimeEncoder() + exp := make([]time.Time, len(values)) + for i, v := range values { + exp[i] = time.Unix(0, v) + enc.Write(exp[i]) + } + + // Retrieve encoded bytes from encoder. + buf, err := enc.Bytes() + if err != nil { + t.Fatal(err) + } + + // Read values out of decoder. + got := make([]time.Time, 0, len(values)) + dec := NewTimeDecoder(buf) + for dec.Next() { + if err := dec.Error(); err != nil { + t.Fatal(err) + } + got = append(got, dec.Read()) + } + + // Verify that input and output values match. + if !reflect.DeepEqual(exp, got) { + t.Fatalf("mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", exp, got) + } + + return true + }, nil) +} + func BenchmarkTimeEncoder(b *testing.B) { enc := NewTimeEncoder() x := make([]time.Time, 1024) diff --git a/tsdb/engine/tsm1/write_lock_test.go b/tsdb/engine/tsm1/write_lock_test.go index 7fa17c530c..30178b8fcd 100644 --- a/tsdb/engine/tsm1/write_lock_test.go +++ b/tsdb/engine/tsm1/write_lock_test.go @@ -1,8 +1,9 @@ package tsm1_test import ( - // "sync" + "sync" "testing" + "testing/quick" "time" "github.com/influxdb/influxdb/tsdb/engine/tsm1" @@ -129,3 +130,37 @@ func TestWriteLock_Same(t *testing.T) { // // we're all good // } // } + +func TestWriteLock_Quick(t *testing.T) { + if testing.Short() { + t.Skip("short mode") + } + + quick.Check(func(extents []struct{ Min, Max uint64 }) bool { + var wg sync.WaitGroup + var mu tsm1.WriteLock + for _, extent := range extents { + // Limit range. + extent.Min %= 10 + extent.Max %= 10 + + // Reverse if out of order. + if extent.Min > extent.Max { + extent.Min, extent.Max = extent.Max, extent.Min + } + + // Lock, wait and unlock in a separate goroutine. + wg.Add(1) + go func(min, max int64) { + defer wg.Done() + mu.LockRange(min, max) + time.Sleep(1 * time.Millisecond) + mu.UnlockRange(min, max) + }(int64(extent.Min), int64(extent.Max)) + } + + // All locks should return. + wg.Wait() + return true + }, nil) +}