Fix direct index size not calculated

tsdb/engine/tsm1/compact.go

@@ -200,14 +200,11 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup {
 	generations := c.findGenerations()
 
 	// first check if we should be doing a full compaction because nothing has been written in a long time
-	if !c.lastPlanCompactedFull && c.CompactFullWriteColdDuration > 0 && time.Now().Sub(lastWrite) > c.CompactFullWriteColdDuration {
+	if !c.lastPlanCompactedFull && c.CompactFullWriteColdDuration > 0 && time.Now().Sub(lastWrite) > c.CompactFullWriteColdDuration && len(generations) > 1 {
 		var tsmFiles []string
 		for _, group := range generations {
-			// If the generation size is less the max size
-			if group.size() < uint64(maxTSMFileSize) || group.hasTombstones() {
-				for _, f := range group.files {
-					tsmFiles = append(tsmFiles, f.Path)
-				}
+			for _, f := range group.files {
+				tsmFiles = append(tsmFiles, f.Path)
 			}
 		}
 		sort.Strings(tsmFiles)

tsdb/engine/tsm1/compact_test.go

@@ -848,46 +848,6 @@ func TestDefaultPlanner_Plan_FullOnCold(t *testing.T) {
 	}
 }
 
-// Ensure that the planner will compact all files if no writes
-// have happened in some interval but skip files already over the limit
-func TestDefaultPlanner_Plan_FullSkipMaxSize(t *testing.T) {
-	data := []tsm1.FileStat{
-		tsm1.FileStat{
-			Path: "01-01.tsm1",
-			Size: 2049 * 1024 * 1024,
-		},
-		tsm1.FileStat{
-			Path: "02-02.tsm1",
-			Size: 129 * 1024 * 1024,
-		},
-		tsm1.FileStat{
-			Path: "03-02.tsm1",
-			Size: 33 * 1024 * 1024,
-		},
-	}
-
-	cp := &tsm1.DefaultPlanner{
-		FileStore: &fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		},
-		CompactFullWriteColdDuration: time.Nanosecond,
-	}
-
-	expFiles := []tsm1.FileStat{data[1], data[2]}
-	tsm := cp.Plan(time.Now().Add(-time.Second))
-	if exp, got := len(expFiles), len(tsm[0]); got != exp {
-		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
-	}
-
-	for i, p := range expFiles {
-		if got, exp := tsm[0][i], p.Path; got != exp {
-			t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
-		}
-	}
-}
-
 // Ensure that the planner will not return files that are over the max
 // allowable size
 func TestDefaultPlanner_Plan_SkipMaxSizeFiles(t *testing.T) {

tsdb/engine/tsm1/writer.go

@@ -233,8 +233,8 @@ func NewDirectIndex() TSMIndex {
 // directIndex is a simple in-memory index implementation for a TSM file.  The full index
 // must fit in memory.
 type directIndex struct {
-	mu sync.RWMutex
-
+	mu     sync.RWMutex
+	size   uint32
 	blocks map[string]*indexEntries
 }
 
@@ -248,6 +248,11 @@ func (d *directIndex) Add(key string, blockType byte, minTime, maxTime time.Time
 			Type: blockType,
 		}
 		d.blocks[key] = entries
+		// size of the key stored in the index
+		d.size += uint32(2 + len(key))
+
+		// size of the count of entries stored in the index
+		d.size += indexCountSize
 	}
 	entries.Append(&IndexEntry{
 		MinTime: minTime,
@@ -255,6 +260,10 @@ func (d *directIndex) Add(key string, blockType byte, minTime, maxTime time.Time
 		Offset:  offset,
 		Size:    size,
 	})
+
+	// size of the encoded index entry
+	d.size += indexEntrySize
+
 }
 
 func (d *directIndex) Entries(key string) []*IndexEntry {
@@ -432,6 +441,8 @@ func (d *directIndex) UnmarshalBinary(b []byte) error {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 
+	d.size = uint32(len(b))
+
 	var pos int
 	for pos < len(b) {
 		n, key, err := readKey(b[pos:])
@@ -452,7 +463,7 @@ func (d *directIndex) UnmarshalBinary(b []byte) error {
 }
 
 func (d *directIndex) Size() uint32 {
-	return 0
+	return d.size
 }
 
 // tsmWriter writes keys and values in the TSM format