Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor series file tombstoning.

pull/9150/head
Ben Johnson 2017-10-26 13:55:00 -06:00
parent 3034d3fb54
commit d3cd750509
No known key found for this signature in database
GPG Key ID: 81741CD251883081
8 changed files with 131 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
tsdb/index/tsi1/file_set.go
View File

@ -204,7 +204,7 @@ func (fs *FileSet) MeasurementSeriesIDIterator(name []byte) SeriesIDIterator {
a = append(a, itr)
}
}
return FilterUndeletedSeriesIDIterator(MergeSeriesIDIterators(a...))
return FilterUndeletedSeriesIDIterator(fs.sfile, MergeSeriesIDIterators(a...))
}
// TagKeyIterator returns an iterator over all tag keys for a measurement.
@ -383,7 +383,7 @@ func (fs *FileSet) TagKeySeriesIDIterator(name, key []byte) SeriesIDIterator {
a = append(a, itr)
}
}
return FilterUndeletedSeriesIDIterator(MergeSeriesIDIterators(a...))
return FilterUndeletedSeriesIDIterator(fs.sfile, MergeSeriesIDIterators(a...))
}
// HasTagKey returns true if the tag key exists.
@ -427,7 +427,7 @@ func (fs *FileSet) TagValueSeriesIDIterator(name, key, value []byte) SeriesIDIte
a = append(a, itr)
}
}
return FilterUndeletedSeriesIDIterator(MergeSeriesIDIterators(a...))
return FilterUndeletedSeriesIDIterator(fs.sfile, MergeSeriesIDIterators(a...))
}
// MatchTagValueSeriesIDIterator returns a series iterator for tags which match value.
@ -437,15 +437,15 @@ func (fs *FileSet) MatchTagValueSeriesIDIterator(name, key []byte, value *regexp
if matches {
if matchEmpty {
return FilterUndeletedSeriesIDIterator(fs.matchTagValueEqualEmptySeriesIDIterator(name, key, value))
return FilterUndeletedSeriesIDIterator(fs.sfile, fs.matchTagValueEqualEmptySeriesIDIterator(name, key, value))
}
return FilterUndeletedSeriesIDIterator(fs.matchTagValueEqualNotEmptySeriesIDIterator(name, key, value))
return FilterUndeletedSeriesIDIterator(fs.sfile, fs.matchTagValueEqualNotEmptySeriesIDIterator(name, key, value))
}
if matchEmpty {
return FilterUndeletedSeriesIDIterator(fs.matchTagValueNotEqualEmptySeriesIDIterator(name, key, value))
return FilterUndeletedSeriesIDIterator(fs.sfile, fs.matchTagValueNotEqualEmptySeriesIDIterator(name, key, value))
}
return FilterUndeletedSeriesIDIterator(fs.matchTagValueNotEqualNotEmptySeriesIDIterator(name, key, value))
return FilterUndeletedSeriesIDIterator(fs.sfile, fs.matchTagValueNotEqualNotEmptySeriesIDIterator(name, key, value))
}
func (fs *FileSet) matchTagValueEqualEmptySeriesIDIterator(name, key []byte, value *regexp.Regexp) SeriesIDIterator {
@ -678,19 +678,6 @@ func (fs *FileSet) measurementNamesByTagFilter(op influxql.Token, key, val strin
return names
}
// SeriesSketches returns the merged series sketches for the FileSet.
func (fs *FileSet) SeriesSketches() (estimator.Sketch, estimator.Sketch, error) {
sketch, tsketch := hll.NewDefaultPlus(), hll.NewDefaultPlus()
// Iterate over all the files and merge the sketches into the result.
for _, f := range fs.files {
if err := f.MergeSeriesSketches(sketch, tsketch); err != nil {
return nil, nil, err
}
}
return sketch, tsketch, nil
}
// MeasurementsSketches returns the merged measurement sketches for the FileSet.
func (fs *FileSet) MeasurementsSketches() (estimator.Sketch, estimator.Sketch, error) {
sketch, tsketch := hll.NewDefaultPlus(), hll.NewDefaultPlus()
@ -919,7 +906,7 @@ type File interface {
TagValueSeriesIDIterator(name, key, value []byte) SeriesIDIterator
// Sketches for cardinality estimation
MergeSeriesSketches(s, t estimator.Sketch) error
// MergeSeriesSketches(s, t estimator.Sketch) error
MergeMeasurementsSketches(s, t estimator.Sketch) error
// Reference counting.

9
tsdb/index/tsi1/index_file.go
View File

@ -315,15 +315,6 @@ func (f *IndexFile) MeasurementSeriesIDIterator(name []byte) SeriesIDIterator {
return f.mblk.SeriesIDIterator(name)
}
// MergeSeriesSketches merges the index file's series sketches into the provided
// sketches.
func (f *IndexFile) MergeSeriesSketches(s, t estimator.Sketch) error {
if err := s.Merge(f.sSketch); err != nil {
return err
}
return t.Merge(f.sTSketch)
}
// MergeMeasurementsSketches merges the index file's series sketches into the provided
// sketches.
func (f *IndexFile) MergeMeasurementsSketches(s, t estimator.Sketch) error {

33
tsdb/index/tsi1/log_file.go
View File

@ -593,7 +593,7 @@ func (f *LogFile) execDeleteMeasurementEntry(e *LogEntry) {
mm := f.createMeasurementIfNotExists(e.Name)
mm.deleted = true
mm.tagSet = make(map[string]logTagKey)
mm.series = make(map[uint64]bool)
mm.series = make(map[uint64]struct{})
// Update measurement tombstone sketch.
f.mTSketch.Add(e.Name)
@ -629,6 +629,7 @@ func (f *LogFile) execSeriesEntry(e *LogEntry) {
// Read measurement name.
name, remainder := ReadSeriesKeyMeasurement(remainder)
mm := f.createMeasurementIfNotExists(name)
mm.series[e.SeriesID] = struct{}{}
// Read tag count.
tagN, remainder := ReadSeriesKeyTagN(remainder)
@ -709,7 +710,7 @@ func (f *LogFile) createMeasurementIfNotExists(name []byte) *logMeasurement {
mm = &logMeasurement{
name: name,
tagSet: make(map[string]logTagKey),
series: make(map[uint64]bool),
series: make(map[uint64]struct{}),
}
f.mms[string(name)] = mm
}
@ -885,20 +886,6 @@ type logFileMeasurementCompactInfo struct {
size int64
}
// MergeSeriesSketches merges the series sketches belonging to this LogFile
// into the provided sketches.
//
// MergeSeriesSketches is safe for concurrent use by multiple goroutines.
func (f *LogFile) MergeSeriesSketches(sketch, tsketch estimator.Sketch) error {
f.mu.RLock()
defer f.mu.RUnlock()
if err := sketch.Merge(f.sSketch); err != nil {
return err
}
return tsketch.Merge(f.sTSketch)
}
// MergeMeasurementsSketches merges the measurement sketches belonging to this
// LogFile into the provided sketches.
//
@ -1036,6 +1023,7 @@ func appendLogEntry(dst []byte, e *LogEntry) []byte {
return dst
}
/*
type logSerie struct {
name []byte
tags models.Tags
@ -1063,6 +1051,7 @@ func (a logSeries) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a logSeries) Less(i, j int) bool {
return a[i].Compare(a[j].name, a[j].tags) == -1
}
*/
// logMeasurements represents a map of measurement names to measurements.
type logMeasurements map[string]*logMeasurement
@ -1081,7 +1070,7 @@ type logMeasurement struct {
name []byte
tagSet map[string]logTagKey
deleted bool
series map[uint64]struct{}{}
series map[uint64]struct{}
}
func (mm *logMeasurement) seriesIDs() []uint64 {
@ -1155,7 +1144,7 @@ func (tk *logTagKey) TagValueIterator() TagValueIterator {
func (tk *logTagKey) createTagValueIfNotExists(value []byte) logTagValue {
tv, ok := tk.tagValues[string(value)]
if !ok {
tv = logTagValue{name: value, series: make(map[uint64]bool)}
tv = logTagValue{name: value, series: make(map[uint64]struct{})}
}
return tv
}
@ -1170,7 +1159,7 @@ func (a logTagKeySlice) Less(i, j int) bool { return bytes.Compare(a[i].name, a[
type logTagValue struct {
name []byte
deleted bool
series map[uint64]struct{}{}
series map[uint64]struct{}
}
func (tv *logTagValue) seriesIDs() []uint64 {
@ -1239,14 +1228,14 @@ type logSeriesIDIterator struct {
// newLogSeriesIDIterator returns a new instance of logSeriesIDIterator.
// All series are copied to the iterator.
func newLogSeriesIDIterator(m map[uint64]bool) *logSeriesIDIterator {
func newLogSeriesIDIterator(m map[uint64]struct{}) *logSeriesIDIterator {
if len(m) == 0 {
return nil
}
itr := logSeriesIDIterator{series: make([]SeriesIDElem, 0, len(m))}
for seriesID, deleted := range m {
itr.series = append(itr.series, SeriesIDElem{SeriesID: seriesID, Deleted: deleted})
for seriesID := range m {
itr.series = append(itr.series, SeriesIDElem{SeriesID: seriesID})
}
sort.Sort(SeriesIDElems(itr.series))

4
tsdb/index/tsi1/partition.go
View File

@ -466,7 +466,7 @@ func (i *Partition) DropMeasurement(name []byte) error {
// Delete all series in measurement.
if sitr := fs.MeasurementSeriesIDIterator(name); sitr != nil {
for s := sitr.Next(); s.SeriesID != 0; s = sitr.Next() {
if !s.Deleted {
if !fs.SeriesFile().IsDeleted(s.SeriesID) {
if err := func() error {
i.mu.RLock()
defer i.mu.RUnlock()
@ -538,7 +538,7 @@ func (i *Partition) DropSeries(key []byte) error {
defer fs.Release()
// Check if that was the last series for the measurement in the entire index.
itr := fs.MeasurementSeriesIDIterator(mname)
itr := FilterUndeletedSeriesIDIterator(i.sfile, fs.MeasurementSeriesIDIterator(mname))
if itr == nil {
return nil
} else if e := itr.Next(); e.SeriesID != 0 {

136
tsdb/index/tsi1/series_file.go
View File

@ -22,12 +22,8 @@ const SeriesIDSize = 8
// Series flag constants.
const (
// Marks the series as having been deleted.
SeriesTombstoneFlag = 0x01
// Marks the following bytes as a hash index.
// These bytes should be skipped by an iterator.
SeriesHashIndexFlag = 0x02
SeriesFileFlagSize = 1
SeriesFileTombstoneFlag = 0x01
)
const DefaultMaxSeriesFileSize = 32 * (1 << 30) // 32GB
@ -50,7 +46,6 @@ type SeriesFile struct {
seriesMap *seriesMap
compactingSeriesMap *seriesMap
tombstones map[uint64]struct{}
// MaxSize is the maximum size of the file.
MaxSize int64
@ -59,9 +54,8 @@ type SeriesFile struct {
// NewSeriesFile returns a new instance of SeriesFile.
func NewSeriesFile(path string) *SeriesFile {
return &SeriesFile{
path: path,
tombstones: make(map[uint64]struct{}),
MaxSize: DefaultMaxSeriesFileSize,
path: path,
MaxSize: DefaultMaxSeriesFileSize,
}
}
@ -152,10 +146,11 @@ func (f *SeriesFile) CreateSeriesListIfNotExists(names [][]byte, tagsSlice []mod
f.mu.RLock()
offsets = make([]uint64, len(names))
for i := range names {
offsets[i] = f.offset(names[i], tagsSlice[i], buf)
if offsets[i] == 0 {
offset := f.offset(names[i], tagsSlice[i], buf)
if offset == 0 {
createRequired = true
}
offsets[i] = offset
}
f.mu.RUnlock()
@ -173,7 +168,7 @@ func (f *SeriesFile) CreateSeriesListIfNotExists(names [][]byte, tagsSlice []mod
for i := range names {
// Skip series that have already been created.
if offsets[i] != 0 {
if offset := offsets[i]; offset != 0 {
continue
}
@ -187,6 +182,12 @@ func (f *SeriesFile) CreateSeriesListIfNotExists(names [][]byte, tagsSlice []mod
continue
}
// Append flag byte.
if _, err := f.w.Write([]byte{0}); err != nil {
return nil, err
}
f.size += SeriesFileFlagSize
// Append series to the end of the file.
offset := uint64(f.size)
if _, err := f.w.Write(buf); err != nil {
@ -229,13 +230,41 @@ func (f *SeriesFile) CreateSeriesListIfNotExists(names [][]byte, tagsSlice []mod
return offsets, nil
}
// DeleteSeries flags a series as permanently deleted.
// DeleteSeriesID flags a series as permanently deleted.
// If the series is reintroduced later then it must create a new offset.
func (f *SeriesFile) DeleteSeries(offset uint64) error {
f.tombstones[offset] = struct{}{}
func (f *SeriesFile) DeleteSeriesID(offset uint64) error {
f.mu.Lock()
defer f.mu.Unlock()
// Already tombstoned, ignore.
if _, ok := f.seriesMap.tombstones[offset]; ok {
return nil
}
// Write tombstone entry.
if _, err := f.w.Write([]byte{SeriesFileTombstoneFlag}); err != nil {
return err
} else if err := binary.Write(f.w, binary.BigEndian, offset); err != nil {
return err
} else if err := f.w.Flush(); err != nil {
return err
}
f.size += SeriesFileTombstoneFlag + 8
// Mark tombstone in memory.
f.seriesMap.tombstones[offset] = struct{}{}
return nil
}
// IsDeleted returns true if the ID has been deleted before.
func (f *SeriesFile) IsDeleted(offset uint64) bool {
f.mu.RLock()
_, ok := f.seriesMap.tombstones[offset]
f.mu.RUnlock()
return ok
}
// Offset returns the byte offset of the series within the block.
func (f *SeriesFile) Offset(name []byte, tags models.Tags, buf []byte) (offset uint64) {
f.mu.RLock()
@ -248,7 +277,7 @@ func (f *SeriesFile) offset(name []byte, tags models.Tags, buf []byte) uint64 {
offset := f.seriesMap.offset(AppendSeriesKey(buf[:0], name, tags))
if offset == 0 {
return 0
} else if _, ok := f.tombstones[offset]; ok {
} else if _, ok := f.seriesMap.tombstones[offset]; ok {
return 0
}
return offset
@ -344,16 +373,29 @@ type seriesFileIterator struct {
// Next returns the next series element.
func (itr *seriesFileIterator) Next() SeriesIDElem {
if len(itr.data) == 0 {
return SeriesIDElem{}
for {
if len(itr.data) == 0 {
return SeriesIDElem{}
}
// Read flag.
flag := itr.data[0]
itr.data = itr.data[1:]
itr.offset++
switch flag {
case SeriesFileTombstoneFlag:
itr.data = itr.data[8:] // skip
itr.offset += 8
default:
var key []byte
key, itr.data = ReadSeriesKey(itr.data)
elem := SeriesIDElem{SeriesID: itr.offset}
itr.offset += uint64(len(key))
return elem
}
}
var key []byte
key, itr.data = ReadSeriesKey(itr.data)
elem := SeriesIDElem{SeriesID: itr.offset}
itr.offset += uint64(len(key))
return elem
}
// AppendSeriesKey serializes name and tags to a byte slice.
@ -533,9 +575,10 @@ const (
// seriesMap represents a read-only hash map of series offsets.
type seriesMap struct {
path string
sfile *SeriesFile
inmem *rhh.HashMap
path string
sfile *SeriesFile
inmem *rhh.HashMap
tombstones map[uint64]struct{}
n int64
maxOffset uint64
@ -545,7 +588,11 @@ type seriesMap struct {
}
func newSeriesMap(path string, sfile *SeriesFile) *seriesMap {
return &seriesMap{path: path, sfile: sfile}
return &seriesMap{
path: path,
sfile: sfile,
tombstones: make(map[uint64]struct{}),
}
}
func (m *seriesMap) open() error {
@ -569,13 +616,26 @@ func (m *seriesMap) open() error {
// Index all data created after the on-disk hash map.
inmem := rhh.NewHashMap(rhh.DefaultOptions)
tombstones := make(map[uint64]struct{})
for b, offset := m.sfile.data[m.maxOffset:m.sfile.size], m.maxOffset; len(b) > 0; {
var key []byte
key, b = ReadSeriesKey(b)
inmem.Put(key, offset)
offset += uint64(len(key))
// Read flag.
flag := b[0]
b, offset = b[1:], offset+1
switch flag {
case SeriesFileTombstoneFlag:
seriesID := binary.BigEndian.Uint64(b[:8])
b = b[8:]
tombstones[seriesID] = struct{}{}
default:
var key []byte
key, b = ReadSeriesKey(b)
inmem.Put(key, offset)
offset += uint64(len(key))
}
}
m.inmem = inmem
m.tombstones = tombstones
return nil
}
@ -592,11 +652,13 @@ func (m *seriesMap) close() error {
// offset finds the series key's offset in either the on-disk or in-memory hash maps.
func (m *seriesMap) offset(key []byte) uint64 {
if offset := m.onDiskOffset(key); offset != 0 {
offset, _ := m.inmem.Get(key).(uint64)
if _, ok := m.tombstones[offset]; ok {
return 0
} else if offset != 0 {
return offset
}
offset, _ := m.inmem.Get(key).(uint64)
return offset
return m.onDiskOffset(key)
}
func (m *seriesMap) onDiskOffset(key []byte) uint64 {

10
tsdb/index/tsi1/tsi1.go
View File

@ -342,7 +342,6 @@ func (p tagValueMergeElem) Deleted() bool {
type SeriesIDElem struct {
SeriesID uint64
Deleted bool
Expr influxql.Expr
}
@ -668,15 +667,16 @@ func (itr *seriesIDDifferenceIterator) Next() SeriesIDElem {
// filterUndeletedSeriesIDIterator returns all series which are not deleted.
type filterUndeletedSeriesIDIterator struct {
itr SeriesIDIterator
sfile *SeriesFile
itr SeriesIDIterator
}
// FilterUndeletedSeriesIDIterator returns an iterator which filters all deleted series.
func FilterUndeletedSeriesIDIterator(itr SeriesIDIterator) SeriesIDIterator {
func FilterUndeletedSeriesIDIterator(sfile *SeriesFile, itr SeriesIDIterator) SeriesIDIterator {
if itr == nil {
return nil
}
return &filterUndeletedSeriesIDIterator{itr: itr}
return &filterUndeletedSeriesIDIterator{sfile: sfile, itr: itr}
}
func (itr *filterUndeletedSeriesIDIterator) Next() SeriesIDElem {
@ -684,7 +684,7 @@ func (itr *filterUndeletedSeriesIDIterator) Next() SeriesIDElem {
e := itr.itr.Next()
if e.SeriesID == 0 {
return SeriesIDElem{}
} else if e.Deleted {
} else if itr.sfile.IsDeleted(e.SeriesID) {
continue
}
return e

8
tsdb/index/tsi1/tsi1_test.go
View File

@ -152,7 +152,7 @@ func TestMergeTagValueIterators(t *testing.T) {
// Ensure iterator can operate over an in-memory list of series.
func TestSeriesIDIterator(t *testing.T) {
elems := []tsi1.SeriesIDElem{
{SeriesID: 1, Deleted: true},
{SeriesID: 1},
{SeriesID: 2},
}
@ -171,21 +171,21 @@ func TestMergeSeriesIDIterators(t *testing.T) {
itr := tsi1.MergeSeriesIDIterators(
&SeriesIDIterator{Elems: []tsi1.SeriesIDElem{
{SeriesID: 1},
{SeriesID: 2, Deleted: true},
{SeriesID: 2},
{SeriesID: 3},
}},
&SeriesIDIterator{},
&SeriesIDIterator{Elems: []tsi1.SeriesIDElem{
{SeriesID: 1},
{SeriesID: 2},
{SeriesID: 3, Deleted: true},
{SeriesID: 3},
{SeriesID: 4},
}},
)
if e := itr.Next(); !reflect.DeepEqual(e, tsi1.SeriesIDElem{SeriesID: 1}) {
t.Fatalf("unexpected elem(0): %#v", e)
} else if e := itr.Next(); !reflect.DeepEqual(e, tsi1.SeriesIDElem{SeriesID: 2, Deleted: true}) {
} else if e := itr.Next(); !reflect.DeepEqual(e, tsi1.SeriesIDElem{SeriesID: 2}) {
t.Fatalf("unexpected elem(1): %#v", e)
} else if e := itr.Next(); !reflect.DeepEqual(e, tsi1.SeriesIDElem{SeriesID: 3}) {
t.Fatalf("unexpected elem(2): %#v", e)

2
tsdb/store_test.go
View File

@ -591,6 +591,8 @@ func testStoreCardinalityTombstoning(t *testing.T, store *Store) {
}
func TestStore_Cardinality_Tombstoning(t *testing.T) {
t.Skip("TODO(benbjohnson): Fix once series file moved to DB")
t.Parallel()
if testing.Short() || os.Getenv("GORACE") != "" || os.Getenv("APPVEYOR") != "" {