1621 lines
38 KiB
Go
1621 lines
38 KiB
Go
package tsm1
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"testing"
|
|
)
|
|
|
|
func fatal(t testing.TB, msg string, err error) {
|
|
t.Helper()
|
|
t.Fatalf("unexpected error %v: %v", msg, err)
|
|
}
|
|
|
|
func fatalIfErr(t testing.TB, msg string, err error) {
|
|
t.Helper()
|
|
if err != nil {
|
|
fatal(t, msg, err)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_Type(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
values := []Value{NewValue(0, int64(1))}
|
|
if err := w.Write([]byte("cpu"), values); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
|
|
}
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error opening: %v", err)
|
|
}
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
typ, err := r.Type([]byte("cpu"))
|
|
if err != nil {
|
|
fatal(t, "reading type", err)
|
|
}
|
|
|
|
if got, exp := typ, BlockInteger; got != exp {
|
|
t.Fatalf("type mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestIndexWriter_MaxBlocks(t *testing.T) {
|
|
index := NewIndexWriter()
|
|
for i := 0; i < 1<<16; i++ {
|
|
index.Add([]byte("cpu"), BlockFloat64, 0, 1, 10, 20)
|
|
}
|
|
|
|
if _, err := index.MarshalBinary(); err == nil {
|
|
t.Fatalf("expected max block count error. got nil")
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_ReadAll(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
var data = map[string][]Value{
|
|
"float": []Value{NewValue(1, 1.0)},
|
|
"int": []Value{NewValue(1, int64(1))},
|
|
"uint": []Value{NewValue(1, ^uint64(0))},
|
|
"bool": []Value{NewValue(1, true)},
|
|
"string": []Value{NewValue(1, "foo")},
|
|
}
|
|
|
|
keys := make([]string, 0, len(data))
|
|
for k := range data {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
for _, k := range keys {
|
|
if err := w.Write([]byte(k), data[k]); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
var count int
|
|
for k, vals := range data {
|
|
readValues, err := r.ReadAll([]byte(k))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error readin: %v", err)
|
|
}
|
|
|
|
if exp := len(vals); exp != len(readValues) {
|
|
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
|
|
}
|
|
|
|
for i, v := range vals {
|
|
if v.Value() != readValues[i].Value() {
|
|
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
|
|
}
|
|
}
|
|
count++
|
|
}
|
|
|
|
if got, exp := count, len(data); got != exp {
|
|
t.Fatalf("read values count mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_Read(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
var data = map[string][]Value{
|
|
"float": []Value{
|
|
NewValue(1, 1.0)},
|
|
"int": []Value{
|
|
NewValue(1, int64(1))},
|
|
"uint": []Value{
|
|
NewValue(1, ^uint64(0))},
|
|
"bool": []Value{
|
|
NewValue(1, true)},
|
|
"string": []Value{
|
|
NewValue(1, "foo")},
|
|
}
|
|
|
|
keys := make([]string, 0, len(data))
|
|
for k := range data {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
for _, k := range keys {
|
|
if err := w.Write([]byte(k), data[k]); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
var count int
|
|
for k, vals := range data {
|
|
readValues, err := r.Read([]byte(k), vals[0].UnixNano())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error readin: %v", err)
|
|
}
|
|
|
|
if exp := len(vals); exp != len(readValues) {
|
|
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
|
|
}
|
|
|
|
for i, v := range vals {
|
|
if v.Value() != readValues[i].Value() {
|
|
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
|
|
}
|
|
}
|
|
count++
|
|
}
|
|
|
|
if got, exp := count, len(data); got != exp {
|
|
t.Fatalf("read values count mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_Keys(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
var data = map[string][]Value{
|
|
"float": []Value{
|
|
NewValue(1, 1.0)},
|
|
"int": []Value{
|
|
NewValue(1, int64(1))},
|
|
"uint": []Value{
|
|
NewValue(1, ^uint64(0))},
|
|
"bool": []Value{
|
|
NewValue(1, true)},
|
|
"string": []Value{
|
|
NewValue(1, "foo")},
|
|
}
|
|
|
|
keys := make([]string, 0, len(data))
|
|
for k := range data {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
for _, k := range keys {
|
|
if err := w.Write([]byte(k), data[k]); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
var count int
|
|
for k, vals := range data {
|
|
readValues, err := r.Read([]byte(k), vals[0].UnixNano())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error readin: %v", err)
|
|
}
|
|
|
|
if exp := len(vals); exp != len(readValues) {
|
|
t.Fatalf("read values length mismatch: got %v, exp %v", len(readValues), exp)
|
|
}
|
|
|
|
for i, v := range vals {
|
|
if v.Value() != readValues[i].Value() {
|
|
t.Fatalf("read value mismatch(%d): got %v, exp %d", i, readValues[i].Value(), v.Value())
|
|
}
|
|
}
|
|
count++
|
|
}
|
|
|
|
if got, exp := count, len(data); got != exp {
|
|
t.Fatalf("read values count mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_Tombstone(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
values := []Value{NewValue(0, 1.0)}
|
|
if err := w.Write([]byte("cpu"), values); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.Write([]byte("mem"), values); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
if err := r.Delete([][]byte{[]byte("mem")}); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
r, err = NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if got, exp := r.KeyCount(), 1; got != exp {
|
|
t.Fatalf("key length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneRange(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 2, math.MaxInt64); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 1), true; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 3), false; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), 1; got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := values[0].String(), expValues[0].String(); got != exp {
|
|
t.Fatalf("value mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneOutsideTimeRange(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 0, 0); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 1), true; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 2), true; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 3), true; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.HasTombstones(), false; got != exp {
|
|
t.Fatalf("HasTombstones mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := len(r.TombstoneFiles()), 0; got != exp {
|
|
t.Fatalf("TombstoneFiles len mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneOutsideKeyRange(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("mem")}, 0, 3); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 1), true; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 2), true; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.MaybeContainsValue([]byte("cpu"), 3), true; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.HasTombstones(), false; got != exp {
|
|
t.Fatalf("HasTombstones mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := len(r.TombstoneFiles()), 0; got != exp {
|
|
t.Fatalf("TombstoneFiles len mismatch: got %v, exp %v", got, exp)
|
|
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneOverlapKeyRange(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
}
|
|
if err := w.Write([]byte("cpu,app=foo,host=server-0#!~#value"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.Write([]byte("cpu,app=foo,host=server-73379#!~#value"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{
|
|
[]byte("cpu,app=foo,host=server-0#!~#value"),
|
|
[]byte("cpu,app=foo,host=server-73379#!~#value"),
|
|
[]byte("cpu,app=foo,host=server-99999#!~#value")},
|
|
math.MinInt64, math.MaxInt64); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if got, exp := r.Contains([]byte("cpu,app=foo,host=server-0#!~#value")), false; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.Contains([]byte("cpu,app=foo,host=server-73379#!~#value")), false; got != exp {
|
|
t.Fatalf("MaybeContainsValue mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.HasTombstones(), true; got != exp {
|
|
t.Fatalf("HasTombstones mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := len(r.TombstoneFiles()), 1; got != exp {
|
|
t.Fatalf("TombstoneFiles len mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneFullRange(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, math.MinInt64, math.MaxInt64); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), 0; got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneFullRangeMultiple(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
expValues1 := []Value{
|
|
NewValue(3, 1.0),
|
|
NewValue(4, 2.0),
|
|
NewValue(5, 3.0),
|
|
}
|
|
|
|
if err := w.Write([]byte("mem"), expValues1); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("mem"), []byte("cpu")}, 0, 3); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
// Make sure everything is deleted
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), 0; got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
values, err = r.ReadAll([]byte("mem"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), 2; got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneMultipleRanges(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
NewValue(4, 4.0),
|
|
NewValue(5, 5.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 2, 2); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 4, 4); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), 3; got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneMultipleRangesFull(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 1, 1); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 2, 2); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
if got, exp := r.KeyCount(), 0; got != exp {
|
|
t.Fatalf("key count mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), 0; got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneMultipleRangesNoOverlap(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
expValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 2.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), expValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 1, 1); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu")}, 3, 3); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
|
|
if got, exp := r.KeyCount(), 1; got != exp {
|
|
t.Fatalf("key count mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), 1; got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_MMAP_TombstoneOutsideRange(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
cpuValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(3, 3.0),
|
|
}
|
|
if err := w.Write([]byte("cpu"), cpuValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
memValues := []Value{
|
|
NewValue(1, 1.0),
|
|
NewValue(2, 2.0),
|
|
NewValue(30, 3.0),
|
|
}
|
|
if err := w.Write([]byte("mem"), memValues); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
if err := r.DeleteRange([][]byte{[]byte("cpu"), []byte("mem")}, 5, math.MaxInt64); err != nil {
|
|
t.Fatalf("unexpected error deleting: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
if got, exp := r.KeyCount(), 2; got != exp {
|
|
t.Fatalf("key count mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := len(r.TombstoneRange([]byte("cpu"), nil)), 0; got != exp {
|
|
t.Fatalf("tombstone range mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), len(cpuValues); got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := len(r.TombstoneRange([]byte("mem"), nil)), 1; got != exp {
|
|
t.Fatalf("tombstone range mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
values, err = r.ReadAll([]byte("mem"))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading all: %v", err)
|
|
}
|
|
|
|
if got, exp := len(values), len(memValues[:2]); got != exp {
|
|
t.Fatalf("values length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
}
|
|
|
|
func TestTSMReader_MMAP_Stats(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
values1 := []Value{NewValue(0, 1.0)}
|
|
if err := w.Write([]byte("cpu"), values1); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
values2 := []Value{NewValue(1, 1.0)}
|
|
if err := w.Write([]byte("mem"), values2); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
stats := r.Stats()
|
|
if got, exp := string(stats.MinKey), "cpu"; got != exp {
|
|
t.Fatalf("min key mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := string(stats.MaxKey), "mem"; got != exp {
|
|
t.Fatalf("max key mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := stats.MinTime, values1[0].UnixNano(); got != exp {
|
|
t.Fatalf("min time mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := stats.MaxTime, values2[0].UnixNano(); got != exp {
|
|
t.Fatalf("max time mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
if got, exp := r.KeyCount(), 2; got != exp {
|
|
t.Fatalf("key length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
// Ensure that we return an error if we try to open a non-tsm file
|
|
func TestTSMReader_VerifiesFileType(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
// write some garbage
|
|
f.Write([]byte{0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77, 0x23, 0xac, 0x99, 0x22, 0x77})
|
|
|
|
_, err := NewTSMReader(f)
|
|
if err == nil {
|
|
t.Fatal("expected error trying to open non-tsm file")
|
|
}
|
|
}
|
|
|
|
func TestIndirectIndex_Entries(t *testing.T) {
|
|
index := NewIndexWriter()
|
|
index.Add([]byte("cpu"), BlockFloat64, 0, 1, 10, 100)
|
|
index.Add([]byte("cpu"), BlockFloat64, 2, 3, 20, 200)
|
|
exp := index.Entries([]byte("cpu"))
|
|
|
|
index.Add([]byte("mem"), BlockFloat64, 0, 1, 10, 100)
|
|
|
|
b, err := index.MarshalBinary()
|
|
if err != nil {
|
|
t.Fatalf("unexpected error marshaling index: %v", err)
|
|
}
|
|
|
|
indirect := NewIndirectIndex()
|
|
if err := indirect.UnmarshalBinary(b); err != nil {
|
|
t.Fatalf("unexpected error unmarshaling index: %v", err)
|
|
}
|
|
|
|
entries, err := indirect.ReadEntries([]byte("cpu"), nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if got, exp := len(entries), len(exp); got != exp {
|
|
t.Fatalf("entries length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
|
|
for i, exp := range exp {
|
|
got := entries[i]
|
|
if exp.MinTime != got.MinTime {
|
|
t.Fatalf("minTime mismatch: got %v, exp %v", got.MinTime, exp.MinTime)
|
|
}
|
|
|
|
if exp.MaxTime != got.MaxTime {
|
|
t.Fatalf("minTime mismatch: got %v, exp %v", got.MaxTime, exp.MaxTime)
|
|
}
|
|
|
|
if exp.Size != got.Size {
|
|
t.Fatalf("size mismatch: got %v, exp %v", got.Size, exp.Size)
|
|
}
|
|
if exp.Offset != got.Offset {
|
|
t.Fatalf("size mismatch: got %v, exp %v", got.Offset, exp.Offset)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDirectIndex_KeyCount(t *testing.T) {
|
|
index := NewIndexWriter()
|
|
index.Add([]byte("cpu"), BlockFloat64, 0, 1, 10, 20)
|
|
index.Add([]byte("cpu"), BlockFloat64, 1, 2, 20, 30)
|
|
index.Add([]byte("mem"), BlockFloat64, 0, 1, 10, 20)
|
|
|
|
// 2 distinct keys
|
|
if got, exp := index.KeyCount(), 2; got != exp {
|
|
t.Fatalf("length mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_UnmarshalBinary_BlockCountOverflow(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
for i := 0; i < 3280; i++ {
|
|
w.Write([]byte("cpu"), []Value{NewValue(int64(i), float64(i))})
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
}
|
|
|
|
func TestCompacted_NotFull(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
values := []Value{NewValue(0, 1.0)}
|
|
if err := w.Write([]byte("cpu"), values); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
|
|
}
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
fd, err := os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(fd)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
|
|
iter := r.BlockIterator()
|
|
if !iter.Next() {
|
|
t.Fatalf("expected next, got false")
|
|
}
|
|
|
|
_, _, _, _, _, block, err := iter.Read()
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading block: %v", err)
|
|
}
|
|
|
|
if got, exp := BlockCount(block), 1; got != exp {
|
|
t.Fatalf("block count mismatch: got %v, exp %v", got, exp)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_File_ReadAll(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
var data = map[string][]Value{
|
|
"float": []Value{
|
|
NewValue(1, 1.0)},
|
|
"int": []Value{
|
|
NewValue(1, int64(1))},
|
|
"uint": []Value{
|
|
NewValue(1, ^uint64(0))},
|
|
"bool": []Value{
|
|
NewValue(1, true)},
|
|
"string": []Value{
|
|
NewValue(1, "foo")},
|
|
}
|
|
|
|
keys := make([]string, 0, len(data))
|
|
for k := range data {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
for _, k := range keys {
|
|
if err := w.Write([]byte(k), data[k]); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
var count int
|
|
for k, vals := range data {
|
|
readValues, err := r.ReadAll([]byte(k))
|
|
if err != nil {
|
|
t.Fatalf("unexpected error reading: %v", err)
|
|
}
|
|
|
|
if exp := len(vals); exp != len(readValues) {
|
|
t.Fatalf("read values length mismatch: exp %v, got %v", exp, len(readValues))
|
|
}
|
|
|
|
for i, v := range vals {
|
|
if exp, got := v.Value(), readValues[i].Value(); exp != got {
|
|
t.Fatalf("read value mismatch(%d): exp %v, got %d", i, v.Value(), readValues[i].Value())
|
|
}
|
|
}
|
|
count++
|
|
}
|
|
|
|
if exp, got := len(data), count; exp != got {
|
|
t.Fatalf("read values count mismatch: exp %v, got %v", exp, got)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_FuzzCrashes(t *testing.T) {
|
|
cases := []string{
|
|
"",
|
|
"\x16\xd1\x16\xd1\x01\x10\x14X\xfb\x03\xac~\x80\xf0\x00\x00\x00I^K" +
|
|
"_\xf0\x00\x00\x00D424259389w\xf0\x00\x00\x00" +
|
|
"o\x93\bO\x10?\xf0\x00\x00\x00\x00\b\x00\xc2_\xff\xd8\x0fX^" +
|
|
"/\xbf\xe8\x00\x00\x00\x00\x00\x01\x00\bctr#!~#n\x00" +
|
|
"\x00\x01\x14X\xfb\xb0\x03\xac~\x80\x14X\xfb\xb1\x00\xd4ܥ\x00\x00" +
|
|
"\x00\x00\x00\x00\x00\x05\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00E",
|
|
"\x16\xd1\x16\xd1\x01\x80'Z\\\x00\v)\x00\x00\x00\x00;\x9a\xca\x00" +
|
|
"\x01\x05\x10?\xf0\x00\x00\x00\x00\x00\x00\xc2_\xff\xd6\x1d\xd4&\xed\v" +
|
|
"\xc5\xf7\xfb\xc0\x00\x00\x00\x00\x00 \x00\x06a#!~#v\x00\x00" +
|
|
"\x01\x00\x00\x00\x00;\x9a\xca\x00\x00\x00\x00\x01*\x05\xf2\x00\x00\x00\x00" +
|
|
"\x00\x00\x00\x00\x00\x00\x00\x00\x002",
|
|
"\x16\xd1\x16\xd1\x01\x80\xf0\x00\x00\x00I^K_\xf0\x00\x00\x00D7" +
|
|
"\nw\xf0\x00\x00\x00o\x93\bO\x10?\xf0\x00\x00\x00\x00\x00\x00\xc2" +
|
|
"_\xff\x14X\xfb\xb0\x03\xac~\x80\x14X\xfb\xb1\x00\xd4ܥ\x00\x00" +
|
|
"\x00\x00\x00\x00\x00\x05\x00\x00\x00@\x00\x00\x00\x00\x00\x00\x00E",
|
|
"\x16\xd1\x16\xd1\x01000000000000000" +
|
|
"00000000000000000000" +
|
|
"0000000000\x00\x000\x00\x0100000" +
|
|
"000\x00\x00\x00\x00\x00\x00\x002",
|
|
"\x16\xd1\x16\xd1\x01",
|
|
"\x16\xd1\x16\xd1\x01\x00\x00o\x93\bO\x10?\xf0\x00\x00\x00\x00X^" +
|
|
"/\xbf\xe8\x00\x00\x00\x00\x00\x01\x00\bctr#!~#n\x00" +
|
|
"\x00\x01\x14X\xfb\xb0\x03\xac~\x80\x14X\xfb\xb1\x00\xd4ܥ\x00\x00" +
|
|
"\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00E",
|
|
}
|
|
|
|
for _, c := range cases {
|
|
func() {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
|
|
filename := filepath.Join(dir, "x.tsm")
|
|
if err := ioutil.WriteFile(filename, []byte(c), 0600); err != nil {
|
|
t.Fatalf("exp no error, got %s", err)
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
|
|
f, err := os.Open(filename)
|
|
if err != nil {
|
|
t.Fatalf("exp no error, got %s", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer r.Close()
|
|
|
|
iter := r.BlockIterator()
|
|
for iter.Next() {
|
|
key, _, _, _, _, _, err := iter.Read()
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
_, _ = r.Type(key)
|
|
|
|
if _, err = r.ReadAll(key); err != nil {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_File_Read(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
var data = map[string][]Value{
|
|
"float": []Value{
|
|
NewValue(1, 1.0)},
|
|
"int": []Value{
|
|
NewValue(1, int64(1))},
|
|
"uint": []Value{
|
|
NewValue(1, ^uint64(0))},
|
|
"bool": []Value{
|
|
NewValue(1, true)},
|
|
"string": []Value{
|
|
NewValue(1, "foo")},
|
|
}
|
|
|
|
keys := make([]string, 0, len(data))
|
|
for k := range data {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
for _, k := range keys {
|
|
if err := w.Write([]byte(k), data[k]); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
var count int
|
|
for k, vals := range data {
|
|
readValues, err := r.Read([]byte(k), vals[0].UnixNano())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error readin: %v", err)
|
|
}
|
|
|
|
if exp, got := len(vals), len(readValues); exp != got {
|
|
t.Fatalf("read values length mismatch: exp %v, got %v", exp, len(readValues))
|
|
}
|
|
|
|
for i, v := range vals {
|
|
if v.Value() != readValues[i].Value() {
|
|
t.Fatalf("read value mismatch(%d): exp %v, got %d", i, v.Value(), readValues[i].Value())
|
|
}
|
|
}
|
|
count++
|
|
}
|
|
|
|
if exp, got := count, len(data); exp != got {
|
|
t.Fatalf("read values count mismatch: exp %v, got %v", exp, got)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_References(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
defer f.Close()
|
|
|
|
w, err := NewTSMWriter(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error creating writer: %v", err)
|
|
}
|
|
|
|
var data = map[string][]Value{
|
|
"float": []Value{
|
|
NewValue(1, 1.0)},
|
|
"int": []Value{
|
|
NewValue(1, int64(1))},
|
|
"uint": []Value{
|
|
NewValue(1, ^uint64(0))},
|
|
"bool": []Value{
|
|
NewValue(1, true)},
|
|
"string": []Value{
|
|
NewValue(1, "foo")},
|
|
}
|
|
|
|
keys := make([]string, 0, len(data))
|
|
for k := range data {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
for _, k := range keys {
|
|
if err := w.Write([]byte(k), data[k]); err != nil {
|
|
t.Fatalf("unexpected error writing: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := w.WriteIndex(); err != nil {
|
|
t.Fatalf("unexpected error writing index: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing: %v", err)
|
|
}
|
|
|
|
f, err = os.Open(f.Name())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error open file: %v", err)
|
|
}
|
|
|
|
r, err := NewTSMReader(f)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error created reader: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
r.Ref()
|
|
|
|
if err := r.Remove(); err != ErrFileInUse {
|
|
t.Fatalf("expected error removing reader: %v", err)
|
|
}
|
|
|
|
var count int
|
|
for k, vals := range data {
|
|
readValues, err := r.Read([]byte(k), vals[0].UnixNano())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error readin: %v", err)
|
|
}
|
|
|
|
if exp, got := len(vals), len(readValues); exp != got {
|
|
t.Fatalf("read values length mismatch: exp %v, got %v", exp, len(readValues))
|
|
}
|
|
|
|
for i, v := range vals {
|
|
if v.Value() != readValues[i].Value() {
|
|
t.Fatalf("read value mismatch(%d): exp %v, got %d", i, v.Value(), readValues[i].Value())
|
|
}
|
|
}
|
|
count++
|
|
}
|
|
|
|
if exp, got := count, len(data); exp != got {
|
|
t.Fatalf("read values count mismatch: exp %v, got %v", exp, got)
|
|
}
|
|
r.Unref()
|
|
|
|
if err := r.Close(); err != nil {
|
|
t.Fatalf("unexpected error closing reader: %v", err)
|
|
}
|
|
|
|
if err := r.Remove(); err != nil {
|
|
t.Fatalf("unexpected error removing reader: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTSMReader_DeletePrefix(t *testing.T) {
|
|
dir := mustTempDir()
|
|
defer os.RemoveAll(dir)
|
|
f := mustTempFile(dir)
|
|
|
|
// create data in a tsm file
|
|
w, err := NewTSMWriter(f)
|
|
fatalIfErr(t, "creating writer", err)
|
|
|
|
err = w.Write([]byte("cpu"), []Value{
|
|
NewValue(0, int64(1)),
|
|
NewValue(5, int64(2)),
|
|
NewValue(10, int64(3)),
|
|
NewValue(15, int64(4)),
|
|
})
|
|
fatalIfErr(t, "writing", err)
|
|
|
|
err = w.WriteIndex()
|
|
fatalIfErr(t, "writing index", err)
|
|
|
|
err = w.Close()
|
|
fatalIfErr(t, "closing", err)
|
|
|
|
// open the tsm file and delete the prefix
|
|
f, err = os.Open(f.Name())
|
|
fatalIfErr(t, "opening", err)
|
|
|
|
r, err := NewTSMReader(f)
|
|
fatalIfErr(t, "creating reader", err)
|
|
|
|
err = r.DeletePrefix([]byte("c"), 0, 5, nil, nil)
|
|
fatalIfErr(t, "deleting prefix", err)
|
|
|
|
values, err := r.ReadAll([]byte("cpu"))
|
|
fatalIfErr(t, "reading values", err)
|
|
if got, exp := len(values), 2; got != exp {
|
|
t.Fatalf("wrong number of values: %d but wanted: %d", got, exp)
|
|
}
|
|
if got, exp := values[0], NewValue(10, int64(3)); got != exp {
|
|
t.Fatalf("wrong value: %q but wanted %q", got, exp)
|
|
}
|
|
if got, exp := values[1], NewValue(15, int64(4)); got != exp {
|
|
t.Fatalf("wrong value: %q but wanted %q", got, exp)
|
|
}
|
|
|
|
err = r.Close()
|
|
fatalIfErr(t, "closing reader", err)
|
|
|
|
// open the tsm file and check that the deletes still happened
|
|
f, err = os.Open(f.Name())
|
|
fatalIfErr(t, "opening", err)
|
|
|
|
r, err = NewTSMReader(f)
|
|
fatalIfErr(t, "creating reader", err)
|
|
|
|
values, err = r.ReadAll([]byte("cpu"))
|
|
fatalIfErr(t, "reading values", err)
|
|
if got, exp := len(values), 2; got != exp {
|
|
t.Fatalf("wrong number of values: %d but wanted: %d", got, exp)
|
|
}
|
|
if got, exp := values[0], NewValue(10, int64(3)); got != exp {
|
|
t.Fatalf("wrong value: %q but wanted %q", got, exp)
|
|
}
|
|
if got, exp := values[1], NewValue(15, int64(4)); got != exp {
|
|
t.Fatalf("wrong value: %q but wanted %q", got, exp)
|
|
}
|
|
}
|