474 lines
12 KiB
Go
474 lines
12 KiB
Go
package tsi1
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sync"
|
|
"unsafe"
|
|
|
|
"github.com/influxdata/platform/models"
|
|
"github.com/influxdata/platform/pkg/mmap"
|
|
"github.com/influxdata/platform/tsdb"
|
|
)
|
|
|
|
// IndexFileVersion is the current TSI1 index file version.
|
|
const IndexFileVersion = 1
|
|
|
|
// FileSignature represents a magic number at the header of the index file.
|
|
const FileSignature = "TSI1"
|
|
|
|
// IndexFile field size constants.
|
|
const (
|
|
// IndexFile trailer fields
|
|
IndexFileVersionSize = 2
|
|
|
|
// IndexFileTrailerSize is the size of the trailer. Currently 82 bytes.
|
|
IndexFileTrailerSize = IndexFileVersionSize +
|
|
8 + 8 + // measurement block offset + size
|
|
8 + 8 + // series id set offset + size
|
|
8 + 8 + // tombstone series id set offset + size
|
|
// legacy sketch info. we used to have HLL sketches, but they were
|
|
// removed. we keep the offset and length bytes in the trailer so
|
|
// that we don't have to do a migration, but they are unused.
|
|
8 + 8 + 8 + 8 +
|
|
0
|
|
)
|
|
|
|
// IndexFile errors.
|
|
var (
|
|
ErrInvalidIndexFile = errors.New("invalid index file")
|
|
ErrUnsupportedIndexFileVersion = errors.New("unsupported index file version")
|
|
)
|
|
|
|
// IndexFile represents a collection of measurement, tag, and series data.
|
|
type IndexFile struct {
|
|
wg sync.WaitGroup // ref count
|
|
data []byte
|
|
|
|
// Components
|
|
sfile *tsdb.SeriesFile
|
|
tblks map[string]*TagBlock // tag blocks by measurement name
|
|
mblk MeasurementBlock
|
|
|
|
// Raw series set data.
|
|
seriesIDSetData []byte
|
|
tombstoneSeriesIDSetData []byte
|
|
|
|
// Sortable identifier & filepath to the log file.
|
|
level int
|
|
id int
|
|
|
|
mu sync.RWMutex
|
|
// Compaction tracking.
|
|
compacting bool
|
|
|
|
// Path to data file.
|
|
path string
|
|
}
|
|
|
|
// NewIndexFile returns a new instance of IndexFile.
|
|
func NewIndexFile(sfile *tsdb.SeriesFile) *IndexFile {
|
|
return &IndexFile{
|
|
sfile: sfile,
|
|
}
|
|
}
|
|
|
|
// bytes estimates the memory footprint of this IndexFile, in bytes.
|
|
func (f *IndexFile) bytes() int {
|
|
var b int
|
|
f.wg.Add(1)
|
|
b += 16 // wg WaitGroup is 16 bytes
|
|
b += int(unsafe.Sizeof(f.data))
|
|
// Do not count f.data contents because it is mmap'd
|
|
b += int(unsafe.Sizeof(f.sfile))
|
|
// Do not count SeriesFile because it belongs to the code that constructed this IndexFile.
|
|
b += int(unsafe.Sizeof(f.tblks))
|
|
for k, v := range f.tblks {
|
|
// Do not count TagBlock contents, they all reference f.data
|
|
b += int(unsafe.Sizeof(k)) + len(k)
|
|
b += int(unsafe.Sizeof(*v))
|
|
}
|
|
b += int(unsafe.Sizeof(f.mblk)) + f.mblk.bytes()
|
|
b += int(unsafe.Sizeof(f.seriesIDSetData) + unsafe.Sizeof(f.tombstoneSeriesIDSetData))
|
|
// Do not count contents of seriesIDSetData or tombstoneSeriesIDSetData: references f.data
|
|
b += int(unsafe.Sizeof(f.level) + unsafe.Sizeof(f.id))
|
|
b += 24 // mu RWMutex is 24 bytes
|
|
b += int(unsafe.Sizeof(f.compacting))
|
|
b += int(unsafe.Sizeof(f.path)) + len(f.path)
|
|
f.wg.Done()
|
|
return b
|
|
}
|
|
|
|
// Open memory maps the data file at the file's path.
|
|
func (f *IndexFile) Open() error {
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
err = fmt.Errorf("[Index file: %s] %v", f.path, err)
|
|
panic(err)
|
|
}
|
|
}()
|
|
|
|
// Extract identifier from path name.
|
|
f.id, f.level = ParseFilename(f.Path())
|
|
|
|
data, err := mmap.Map(f.Path(), 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return f.UnmarshalBinary(data)
|
|
}
|
|
|
|
// Close unmaps the data file.
|
|
func (f *IndexFile) Close() error {
|
|
// Wait until all references are released.
|
|
f.wg.Wait()
|
|
|
|
f.sfile = nil
|
|
f.tblks = nil
|
|
f.mblk = MeasurementBlock{}
|
|
return mmap.Unmap(f.data)
|
|
}
|
|
|
|
// ID returns the file sequence identifier.
|
|
func (f *IndexFile) ID() int { return f.id }
|
|
|
|
// Path returns the file path.
|
|
func (f *IndexFile) Path() string { return f.path }
|
|
|
|
// SetPath sets the file's path.
|
|
func (f *IndexFile) SetPath(path string) { f.path = path }
|
|
|
|
// Level returns the compaction level for the file.
|
|
func (f *IndexFile) Level() int { return f.level }
|
|
|
|
// Retain adds a reference count to the file.
|
|
func (f *IndexFile) Retain() { f.wg.Add(1) }
|
|
|
|
// Release removes a reference count from the file.
|
|
func (f *IndexFile) Release() { f.wg.Done() }
|
|
|
|
// Size returns the size of the index file, in bytes.
|
|
func (f *IndexFile) Size() int64 { return int64(len(f.data)) }
|
|
|
|
// Compacting returns true if the file is being compacted.
|
|
func (f *IndexFile) Compacting() bool {
|
|
f.mu.RLock()
|
|
v := f.compacting
|
|
f.mu.RUnlock()
|
|
return v
|
|
}
|
|
|
|
// UnmarshalBinary opens an index from data.
|
|
// The byte slice is retained so it must be kept open.
|
|
func (f *IndexFile) UnmarshalBinary(data []byte) error {
|
|
// Ensure magic number exists at the beginning.
|
|
if len(data) < len(FileSignature) {
|
|
return io.ErrShortBuffer
|
|
} else if !bytes.Equal(data[:len(FileSignature)], []byte(FileSignature)) {
|
|
return ErrInvalidIndexFile
|
|
}
|
|
|
|
// Read index file trailer.
|
|
t, err := ReadIndexFileTrailer(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Slice series set data.
|
|
f.seriesIDSetData = data[t.SeriesIDSet.Offset : t.SeriesIDSet.Offset+t.SeriesIDSet.Size]
|
|
f.tombstoneSeriesIDSetData = data[t.TombstoneSeriesIDSet.Offset : t.TombstoneSeriesIDSet.Offset+t.TombstoneSeriesIDSet.Size]
|
|
|
|
// Unmarshal measurement block.
|
|
if err := f.mblk.UnmarshalBinary(data[t.MeasurementBlock.Offset:][:t.MeasurementBlock.Size]); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Unmarshal each tag block.
|
|
f.tblks = make(map[string]*TagBlock)
|
|
itr := f.mblk.Iterator()
|
|
|
|
for m := itr.Next(); m != nil; m = itr.Next() {
|
|
e := m.(*MeasurementBlockElem)
|
|
|
|
// Slice measurement block data.
|
|
buf := data[e.tagBlock.offset:]
|
|
buf = buf[:e.tagBlock.size]
|
|
|
|
// Unmarshal measurement block.
|
|
var tblk TagBlock
|
|
if err := tblk.UnmarshalBinary(buf); err != nil {
|
|
return err
|
|
}
|
|
f.tblks[string(e.name)] = &tblk
|
|
}
|
|
|
|
// Save reference to entire data block.
|
|
f.data = data
|
|
|
|
return nil
|
|
}
|
|
|
|
func (f *IndexFile) SeriesIDSet() (*tsdb.SeriesIDSet, error) {
|
|
ss := tsdb.NewSeriesIDSet()
|
|
if err := ss.UnmarshalBinary(f.seriesIDSetData); err != nil {
|
|
return nil, err
|
|
}
|
|
return ss, nil
|
|
}
|
|
|
|
func (f *IndexFile) TombstoneSeriesIDSet() (*tsdb.SeriesIDSet, error) {
|
|
ss := tsdb.NewSeriesIDSet()
|
|
if err := ss.UnmarshalBinaryUnsafe(f.tombstoneSeriesIDSetData); err != nil {
|
|
return nil, err
|
|
}
|
|
return ss, nil
|
|
}
|
|
|
|
// Measurement returns a measurement element.
|
|
func (f *IndexFile) Measurement(name []byte) MeasurementElem {
|
|
e, ok := f.mblk.Elem(name)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return &e
|
|
}
|
|
|
|
// MeasurementN returns the number of measurements in the file.
|
|
func (f *IndexFile) MeasurementN() (n uint64) {
|
|
mitr := f.mblk.Iterator()
|
|
for me := mitr.Next(); me != nil; me = mitr.Next() {
|
|
n++
|
|
}
|
|
return n
|
|
}
|
|
|
|
// MeasurementHasSeries returns true if a measurement has any non-tombstoned series.
|
|
func (f *IndexFile) MeasurementHasSeries(ss *tsdb.SeriesIDSet, name []byte) (ok bool) {
|
|
e, ok := f.mblk.Elem(name)
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
var exists bool
|
|
e.ForEachSeriesID(func(id tsdb.SeriesID) error {
|
|
if ss.Contains(id) {
|
|
exists = true
|
|
return errors.New("done")
|
|
}
|
|
return nil
|
|
})
|
|
return exists
|
|
}
|
|
|
|
// TagValueIterator returns a value iterator for a tag key and a flag
|
|
// indicating if a tombstone exists on the measurement or key.
|
|
func (f *IndexFile) TagValueIterator(name, key []byte) TagValueIterator {
|
|
tblk := f.tblks[string(name)]
|
|
if tblk == nil {
|
|
return nil
|
|
}
|
|
|
|
// Find key element.
|
|
ke := tblk.TagKeyElem(key)
|
|
if ke == nil {
|
|
return nil
|
|
}
|
|
|
|
// Merge all value series iterators together.
|
|
return ke.TagValueIterator()
|
|
}
|
|
|
|
// TagKeySeriesIDIterator returns a series iterator for a tag key and a flag
|
|
// indicating if a tombstone exists on the measurement or key.
|
|
func (f *IndexFile) TagKeySeriesIDIterator(name, key []byte) tsdb.SeriesIDIterator {
|
|
tblk := f.tblks[string(name)]
|
|
if tblk == nil {
|
|
return nil
|
|
}
|
|
|
|
// Find key element.
|
|
ke := tblk.TagKeyElem(key)
|
|
if ke == nil {
|
|
return nil
|
|
}
|
|
|
|
// Merge all value series iterators together.
|
|
vitr := ke.TagValueIterator()
|
|
var itrs []tsdb.SeriesIDIterator
|
|
for ve := vitr.Next(); ve != nil; ve = vitr.Next() {
|
|
sitr := &rawSeriesIDIterator{data: ve.(*TagBlockValueElem).series.data}
|
|
itrs = append(itrs, sitr)
|
|
}
|
|
|
|
return tsdb.MergeSeriesIDIterators(itrs...)
|
|
}
|
|
|
|
// TagValueSeriesIDSet returns a series id set for a tag value.
|
|
func (f *IndexFile) TagValueSeriesIDSet(name, key, value []byte) (*tsdb.SeriesIDSet, error) {
|
|
tblk := f.tblks[string(name)]
|
|
if tblk == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
// Find value element.
|
|
var valueElem TagBlockValueElem
|
|
if !tblk.DecodeTagValueElem(key, value, &valueElem) {
|
|
return nil, nil
|
|
} else if valueElem.SeriesN() == 0 {
|
|
return nil, nil
|
|
}
|
|
return valueElem.SeriesIDSet()
|
|
}
|
|
|
|
// TagKey returns a tag key.
|
|
func (f *IndexFile) TagKey(name, key []byte) TagKeyElem {
|
|
tblk := f.tblks[string(name)]
|
|
if tblk == nil {
|
|
return nil
|
|
}
|
|
return tblk.TagKeyElem(key)
|
|
}
|
|
|
|
// TagValue returns a tag value.
|
|
func (f *IndexFile) TagValue(name, key, value []byte) TagValueElem {
|
|
tblk := f.tblks[string(name)]
|
|
if tblk == nil {
|
|
return nil
|
|
}
|
|
return tblk.TagValueElem(key, value)
|
|
}
|
|
|
|
// HasSeries returns flags indicating if the series exists and if it is tombstoned.
|
|
func (f *IndexFile) HasSeries(name []byte, tags models.Tags, buf []byte) (exists, tombstoned bool) {
|
|
return f.sfile.HasSeries(name, tags, buf), false // TODO(benbjohnson): series tombstone
|
|
}
|
|
|
|
// TagValueElem returns an element for a measurement/tag/value.
|
|
func (f *IndexFile) TagValueElem(name, key, value []byte) TagValueElem {
|
|
tblk, ok := f.tblks[string(name)]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return tblk.TagValueElem(key, value)
|
|
}
|
|
|
|
// MeasurementIterator returns an iterator over all measurements.
|
|
func (f *IndexFile) MeasurementIterator() MeasurementIterator {
|
|
return f.mblk.Iterator()
|
|
}
|
|
|
|
// TagKeyIterator returns an iterator over all tag keys for a measurement.
|
|
func (f *IndexFile) TagKeyIterator(name []byte) TagKeyIterator {
|
|
blk := f.tblks[string(name)]
|
|
if blk == nil {
|
|
return nil
|
|
}
|
|
return blk.TagKeyIterator()
|
|
}
|
|
|
|
// MeasurementSeriesIDIterator returns an iterator over a measurement's series.
|
|
func (f *IndexFile) MeasurementSeriesIDIterator(name []byte) tsdb.SeriesIDIterator {
|
|
return f.mblk.SeriesIDIterator(name)
|
|
}
|
|
|
|
// ReadIndexFileTrailer returns the index file trailer from data.
|
|
func ReadIndexFileTrailer(data []byte) (IndexFileTrailer, error) {
|
|
var t IndexFileTrailer
|
|
|
|
// Read version.
|
|
t.Version = int(binary.BigEndian.Uint16(data[len(data)-IndexFileVersionSize:]))
|
|
if t.Version != IndexFileVersion {
|
|
return t, ErrUnsupportedIndexFileVersion
|
|
}
|
|
|
|
// Slice trailer data.
|
|
buf := data[len(data)-IndexFileTrailerSize:]
|
|
|
|
// Read measurement block info.
|
|
t.MeasurementBlock.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
|
|
t.MeasurementBlock.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
|
|
|
|
// Read series id set info.
|
|
t.SeriesIDSet.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
|
|
t.SeriesIDSet.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
|
|
|
|
// Read series tombstone id set info.
|
|
t.TombstoneSeriesIDSet.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
|
|
t.TombstoneSeriesIDSet.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
|
|
|
|
// Skip over any legacy sketch data.
|
|
buf = buf[8*4:]
|
|
|
|
if len(buf) != 2 { // Version field still in buffer.
|
|
return t, fmt.Errorf("unread %d bytes left unread in trailer", len(buf)-2)
|
|
}
|
|
return t, nil
|
|
}
|
|
|
|
// IndexFileTrailer represents meta data written to the end of the index file.
|
|
type IndexFileTrailer struct {
|
|
Version int
|
|
|
|
MeasurementBlock struct {
|
|
Offset int64
|
|
Size int64
|
|
}
|
|
|
|
SeriesIDSet struct {
|
|
Offset int64
|
|
Size int64
|
|
}
|
|
|
|
TombstoneSeriesIDSet struct {
|
|
Offset int64
|
|
Size int64
|
|
}
|
|
}
|
|
|
|
// WriteTo writes the trailer to w.
|
|
func (t *IndexFileTrailer) WriteTo(w io.Writer) (n int64, err error) {
|
|
// Write measurement block info.
|
|
if err := writeUint64To(w, uint64(t.MeasurementBlock.Offset), &n); err != nil {
|
|
return n, err
|
|
} else if err := writeUint64To(w, uint64(t.MeasurementBlock.Size), &n); err != nil {
|
|
return n, err
|
|
}
|
|
|
|
// Write series id set info.
|
|
if err := writeUint64To(w, uint64(t.SeriesIDSet.Offset), &n); err != nil {
|
|
return n, err
|
|
} else if err := writeUint64To(w, uint64(t.SeriesIDSet.Size), &n); err != nil {
|
|
return n, err
|
|
}
|
|
|
|
// Write tombstone series id set info.
|
|
if err := writeUint64To(w, uint64(t.TombstoneSeriesIDSet.Offset), &n); err != nil {
|
|
return n, err
|
|
} else if err := writeUint64To(w, uint64(t.TombstoneSeriesIDSet.Size), &n); err != nil {
|
|
return n, err
|
|
}
|
|
|
|
// Write legacy sketch info.
|
|
for i := 0; i < 4; i++ {
|
|
if err := writeUint64To(w, 0, &n); err != nil {
|
|
return n, err
|
|
}
|
|
}
|
|
|
|
// Write index file encoding version.
|
|
if err := writeUint16To(w, IndexFileVersion, &n); err != nil {
|
|
return n, err
|
|
}
|
|
|
|
return n, nil
|
|
}
|
|
|
|
// FormatIndexFileName generates an index filename for the given index.
|
|
func FormatIndexFileName(id, level int) string {
|
|
return fmt.Sprintf("L%d-%08d%s", level, id, IndexFileExt)
|
|
}
|