influxdb/tsdb/tsm1/reader_index_iterator.go

115 lines
2.1 KiB
Go
Raw Normal View History

package tsm1
import (
"fmt"
)
// TSMIndexIterator allows one to iterate over the TSM index.
type TSMIndexIterator struct {
b *faultBuffer
n int
d *indirectIndex
iter *readerOffsetsIterator
// if true, don't need to advance iter on the call to Next
first bool
peeked bool
ok bool
err error
offset uint32
eoffset uint32
tsm1: fix remaining issues and add small benchmarks - notice when keys are deleted during iteration and return an error - make sure all the consumers check the error - add some benchmarks for small indexes to compare - allow concurrent readers to flag deletes benchmarks against base: name old time/op new time/op delta IndirectIndex_UnmarshalBinary-8 70.0ms ±17% 71.0ms ±12% ~ (p=1.000 n=8+8) IndirectIndex_DeleteRangeLast-8 1.48µs ± 1% 0.28µs ± 5% -81.29% (p=0.000 n=8+7) IndirectIndex_DeleteRangeFull/Large-8 786ms ± 1% 363ms ± 3% -53.89% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull/Small-8 2.37ms ± 0% 1.14ms ± 3% -52.02% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 384ms ± 2% 188ms ± 3% -51.04% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 470µs ± 1% 190µs ± 1% -59.71% (p=0.000 n=8+7) IndirectIndex_Delete/Large-8 74.0ms ± 1% 128.7ms ± 1% +73.80% (p=0.001 n=7+7) IndirectIndex_Delete/Small-8 142µs ± 1% 130µs ± 1% -8.24% (p=0.000 n=8+8) name old alloc/op new alloc/op delta IndirectIndex_UnmarshalBinary-8 11.6MB ± 0% 11.7MB ± 0% +0.02% (p=0.000 n=8+7) IndirectIndex_DeleteRangeLast-8 3.26kB ± 0% 0.00kB ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 233MB ± 0% 161MB ± 0% -30.75% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 2.13MB ± 0% 1.40MB ± 0% -34.53% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 12.4MB ± 0% 0.4MB ± 0% -96.82% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 120kB ± 0% 0kB ± 0% -99.89% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 4.54kB ± 0% 0.21kB ± 0% -95.26% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 80.0B ± 0% 0.0B ±NaN% -100.00% (p=0.000 n=8+8) name old allocs/op new allocs/op delta IndirectIndex_UnmarshalBinary-8 35.0 ± 0% 42.0 ± 0% +20.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeLast-8 3.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 1.53M ± 0% 0.52M ± 0% -65.98% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 15.2k ± 0% 5.2k ± 0% -65.97% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 620 ± 0% 124 ± 0% -80.00% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 10.0 ± 0% 2.0 ± 0% -80.00% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 246 ± 0% 1 ± 0% -99.59% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 4.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8)
2018-12-27 20:40:38 +00:00
// lazily loaded from offset and eoffset
key []byte
typ byte
entries []IndexEntry
}
// Next advances the iterator and reports if it is still valid.
func (t *TSMIndexIterator) Next() bool {
tsm1: fix remaining issues and add small benchmarks - notice when keys are deleted during iteration and return an error - make sure all the consumers check the error - add some benchmarks for small indexes to compare - allow concurrent readers to flag deletes benchmarks against base: name old time/op new time/op delta IndirectIndex_UnmarshalBinary-8 70.0ms ±17% 71.0ms ±12% ~ (p=1.000 n=8+8) IndirectIndex_DeleteRangeLast-8 1.48µs ± 1% 0.28µs ± 5% -81.29% (p=0.000 n=8+7) IndirectIndex_DeleteRangeFull/Large-8 786ms ± 1% 363ms ± 3% -53.89% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull/Small-8 2.37ms ± 0% 1.14ms ± 3% -52.02% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 384ms ± 2% 188ms ± 3% -51.04% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 470µs ± 1% 190µs ± 1% -59.71% (p=0.000 n=8+7) IndirectIndex_Delete/Large-8 74.0ms ± 1% 128.7ms ± 1% +73.80% (p=0.001 n=7+7) IndirectIndex_Delete/Small-8 142µs ± 1% 130µs ± 1% -8.24% (p=0.000 n=8+8) name old alloc/op new alloc/op delta IndirectIndex_UnmarshalBinary-8 11.6MB ± 0% 11.7MB ± 0% +0.02% (p=0.000 n=8+7) IndirectIndex_DeleteRangeLast-8 3.26kB ± 0% 0.00kB ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 233MB ± 0% 161MB ± 0% -30.75% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 2.13MB ± 0% 1.40MB ± 0% -34.53% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 12.4MB ± 0% 0.4MB ± 0% -96.82% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 120kB ± 0% 0kB ± 0% -99.89% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 4.54kB ± 0% 0.21kB ± 0% -95.26% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 80.0B ± 0% 0.0B ±NaN% -100.00% (p=0.000 n=8+8) name old allocs/op new allocs/op delta IndirectIndex_UnmarshalBinary-8 35.0 ± 0% 42.0 ± 0% +20.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeLast-8 3.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 1.53M ± 0% 0.52M ± 0% -65.98% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 15.2k ± 0% 5.2k ± 0% -65.97% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 620 ± 0% 124 ± 0% -80.00% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 10.0 ± 0% 2.0 ± 0% -80.00% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 246 ± 0% 1 ± 0% -99.59% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 4.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8)
2018-12-27 20:40:38 +00:00
t.d.mu.RLock()
if n := len(t.d.ro.offsets); t.n != n {
t.err, t.ok = fmt.Errorf("Key count changed during iteration"), false
}
if !t.ok || t.err != nil {
tsm1: fix remaining issues and add small benchmarks - notice when keys are deleted during iteration and return an error - make sure all the consumers check the error - add some benchmarks for small indexes to compare - allow concurrent readers to flag deletes benchmarks against base: name old time/op new time/op delta IndirectIndex_UnmarshalBinary-8 70.0ms ±17% 71.0ms ±12% ~ (p=1.000 n=8+8) IndirectIndex_DeleteRangeLast-8 1.48µs ± 1% 0.28µs ± 5% -81.29% (p=0.000 n=8+7) IndirectIndex_DeleteRangeFull/Large-8 786ms ± 1% 363ms ± 3% -53.89% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull/Small-8 2.37ms ± 0% 1.14ms ± 3% -52.02% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 384ms ± 2% 188ms ± 3% -51.04% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 470µs ± 1% 190µs ± 1% -59.71% (p=0.000 n=8+7) IndirectIndex_Delete/Large-8 74.0ms ± 1% 128.7ms ± 1% +73.80% (p=0.001 n=7+7) IndirectIndex_Delete/Small-8 142µs ± 1% 130µs ± 1% -8.24% (p=0.000 n=8+8) name old alloc/op new alloc/op delta IndirectIndex_UnmarshalBinary-8 11.6MB ± 0% 11.7MB ± 0% +0.02% (p=0.000 n=8+7) IndirectIndex_DeleteRangeLast-8 3.26kB ± 0% 0.00kB ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 233MB ± 0% 161MB ± 0% -30.75% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 2.13MB ± 0% 1.40MB ± 0% -34.53% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 12.4MB ± 0% 0.4MB ± 0% -96.82% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 120kB ± 0% 0kB ± 0% -99.89% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 4.54kB ± 0% 0.21kB ± 0% -95.26% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 80.0B ± 0% 0.0B ±NaN% -100.00% (p=0.000 n=8+8) name old allocs/op new allocs/op delta IndirectIndex_UnmarshalBinary-8 35.0 ± 0% 42.0 ± 0% +20.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeLast-8 3.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 1.53M ± 0% 0.52M ± 0% -65.98% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 15.2k ± 0% 5.2k ± 0% -65.97% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 620 ± 0% 124 ± 0% -80.00% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 10.0 ± 0% 2.0 ± 0% -80.00% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 246 ± 0% 1 ± 0% -99.59% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 4.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8)
2018-12-27 20:40:38 +00:00
t.d.mu.RUnlock()
return false
}
if !t.peeked && !t.first {
t.ok = t.iter.Next()
}
if !t.ok {
tsm1: fix remaining issues and add small benchmarks - notice when keys are deleted during iteration and return an error - make sure all the consumers check the error - add some benchmarks for small indexes to compare - allow concurrent readers to flag deletes benchmarks against base: name old time/op new time/op delta IndirectIndex_UnmarshalBinary-8 70.0ms ±17% 71.0ms ±12% ~ (p=1.000 n=8+8) IndirectIndex_DeleteRangeLast-8 1.48µs ± 1% 0.28µs ± 5% -81.29% (p=0.000 n=8+7) IndirectIndex_DeleteRangeFull/Large-8 786ms ± 1% 363ms ± 3% -53.89% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull/Small-8 2.37ms ± 0% 1.14ms ± 3% -52.02% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 384ms ± 2% 188ms ± 3% -51.04% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 470µs ± 1% 190µs ± 1% -59.71% (p=0.000 n=8+7) IndirectIndex_Delete/Large-8 74.0ms ± 1% 128.7ms ± 1% +73.80% (p=0.001 n=7+7) IndirectIndex_Delete/Small-8 142µs ± 1% 130µs ± 1% -8.24% (p=0.000 n=8+8) name old alloc/op new alloc/op delta IndirectIndex_UnmarshalBinary-8 11.6MB ± 0% 11.7MB ± 0% +0.02% (p=0.000 n=8+7) IndirectIndex_DeleteRangeLast-8 3.26kB ± 0% 0.00kB ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 233MB ± 0% 161MB ± 0% -30.75% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 2.13MB ± 0% 1.40MB ± 0% -34.53% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 12.4MB ± 0% 0.4MB ± 0% -96.82% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 120kB ± 0% 0kB ± 0% -99.89% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 4.54kB ± 0% 0.21kB ± 0% -95.26% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 80.0B ± 0% 0.0B ±NaN% -100.00% (p=0.000 n=8+8) name old allocs/op new allocs/op delta IndirectIndex_UnmarshalBinary-8 35.0 ± 0% 42.0 ± 0% +20.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeLast-8 3.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 1.53M ± 0% 0.52M ± 0% -65.98% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 15.2k ± 0% 5.2k ± 0% -65.97% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 620 ± 0% 124 ± 0% -80.00% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 10.0 ± 0% 2.0 ± 0% -80.00% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 246 ± 0% 1 ± 0% -99.59% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 4.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8)
2018-12-27 20:40:38 +00:00
t.d.mu.RUnlock()
return false
}
t.peeked = false
t.first = false
t.offset = t.iter.Offset()
t.eoffset = t.iter.EntryOffset(t.b)
tsm1: fix remaining issues and add small benchmarks - notice when keys are deleted during iteration and return an error - make sure all the consumers check the error - add some benchmarks for small indexes to compare - allow concurrent readers to flag deletes benchmarks against base: name old time/op new time/op delta IndirectIndex_UnmarshalBinary-8 70.0ms ±17% 71.0ms ±12% ~ (p=1.000 n=8+8) IndirectIndex_DeleteRangeLast-8 1.48µs ± 1% 0.28µs ± 5% -81.29% (p=0.000 n=8+7) IndirectIndex_DeleteRangeFull/Large-8 786ms ± 1% 363ms ± 3% -53.89% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull/Small-8 2.37ms ± 0% 1.14ms ± 3% -52.02% (p=0.000 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 384ms ± 2% 188ms ± 3% -51.04% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 470µs ± 1% 190µs ± 1% -59.71% (p=0.000 n=8+7) IndirectIndex_Delete/Large-8 74.0ms ± 1% 128.7ms ± 1% +73.80% (p=0.001 n=7+7) IndirectIndex_Delete/Small-8 142µs ± 1% 130µs ± 1% -8.24% (p=0.000 n=8+8) name old alloc/op new alloc/op delta IndirectIndex_UnmarshalBinary-8 11.6MB ± 0% 11.7MB ± 0% +0.02% (p=0.000 n=8+7) IndirectIndex_DeleteRangeLast-8 3.26kB ± 0% 0.00kB ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 233MB ± 0% 161MB ± 0% -30.75% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 2.13MB ± 0% 1.40MB ± 0% -34.53% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 12.4MB ± 0% 0.4MB ± 0% -96.82% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 120kB ± 0% 0kB ± 0% -99.89% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 4.54kB ± 0% 0.21kB ± 0% -95.26% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 80.0B ± 0% 0.0B ±NaN% -100.00% (p=0.000 n=8+8) name old allocs/op new allocs/op delta IndirectIndex_UnmarshalBinary-8 35.0 ± 0% 42.0 ± 0% +20.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeLast-8 3.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Large-8 1.53M ± 0% 0.52M ± 0% -65.98% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull/Small-8 15.2k ± 0% 5.2k ± 0% -65.97% (p=0.000 n=8+8) IndirectIndex_DeleteRangeFull_Covered/Large-8 620 ± 0% 124 ± 0% -80.00% (p=0.002 n=7+8) IndirectIndex_DeleteRangeFull_Covered/Small-8 10.0 ± 0% 2.0 ± 0% -80.00% (p=0.000 n=8+8) IndirectIndex_Delete/Large-8 246 ± 0% 1 ± 0% -99.59% (p=0.000 n=8+8) IndirectIndex_Delete/Small-8 4.00 ± 0% 0.00 ±NaN% -100.00% (p=0.000 n=8+8)
2018-12-27 20:40:38 +00:00
t.d.mu.RUnlock()
// reset lazy loaded state
t.key = nil
t.typ = 0
t.entries = t.entries[:0]
return true
}
// Peek reports the next key or nil if there is not one or an error happened.
func (t *TSMIndexIterator) Peek() []byte {
if !t.ok || t.err != nil {
return nil
}
if !t.peeked {
t.ok = t.iter.Next()
t.peeked = true
}
if !t.ok {
return nil
}
return t.iter.Key(t.b)
}
// Key reports the current key.
func (t *TSMIndexIterator) Key() []byte {
if t.key == nil {
buf := t.b.access(t.offset, 0)
t.key = readKey(buf)
t.typ = buf[2+len(t.key)]
}
return t.key
}
// Type reports the current type.
func (t *TSMIndexIterator) Type() byte {
if t.key == nil {
buf := t.b.access(t.offset, 0)
t.key = readKey(buf)
t.typ = buf[2+len(t.key)]
}
return t.typ
}
// Entries reports the current list of entries.
func (t *TSMIndexIterator) Entries() []IndexEntry {
if len(t.entries) == 0 {
buf := t.b.access(t.eoffset, 0)
t.entries, t.err = readEntries(buf, t.entries)
}
if t.err != nil {
return nil
}
return t.entries
}
// Err reports if an error stopped the iteration.
func (t *TSMIndexIterator) Err() error {
return t.err
}