From 52bec1f7f6c0014b6f86720d65d6226d70fe82c2 Mon Sep 17 00:00:00 2001 From: Jason Wilder Date: Fri, 4 Dec 2015 11:51:33 -0700 Subject: [PATCH] Change TSM file naming to generation-sequence.tsm --- tsdb/engine/tsm1/compact.go | 6 ++--- tsdb/engine/tsm1/compact_test.go | 2 +- tsdb/engine/tsm1/file_store.go | 38 +++++++++++++++++------------ tsdb/engine/tsm1/file_store_test.go | 10 ++++---- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go index 38ba0f0e39..7fcb29bece 100644 --- a/tsdb/engine/tsm1/compact.go +++ b/tsdb/engine/tsm1/compact.go @@ -104,7 +104,7 @@ type Compactor struct { MaxFileSize int FileStore interface { - NextID() int + NextGeneration() int } } @@ -163,10 +163,10 @@ func (c *Compactor) writeNewFiles(iter KeyIterator) ([]string, error) { var files []string for { - currentID := c.FileStore.NextID() + currentID := c.FileStore.NextGeneration() // New TSM files are written to a temp file and renamed when fully completed. - fileName := filepath.Join(c.Dir, fmt.Sprintf("%07d.%s.tmp", currentID, "tsm1dev")) + fileName := filepath.Join(c.Dir, fmt.Sprintf("%09d-%09d.%s.tmp", currentID, 1, "tsm1dev")) // Write as much as possible to this file err := c.write(fileName, iter) diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go index 1570b33cf3..f5bcb73cff 100644 --- a/tsdb/engine/tsm1/compact_test.go +++ b/tsdb/engine/tsm1/compact_test.go @@ -519,6 +519,6 @@ func (w *fakeFileStore) Stats() []tsm1.FileStat { return w.PathsFn() } -func (w *fakeFileStore) NextID() int { +func (w *fakeFileStore) NextGeneration() int { return 1 } diff --git a/tsdb/engine/tsm1/file_store.go b/tsdb/engine/tsm1/file_store.go index b7cd6c44ef..d8fabc2470 100644 --- a/tsdb/engine/tsm1/file_store.go +++ b/tsdb/engine/tsm1/file_store.go @@ -69,8 +69,8 @@ type TSMFile interface { type FileStore struct { mu sync.RWMutex - currentFileID int - dir string + currentGeneration int + dir string files []TSMFile } @@ -109,19 +109,19 @@ func (f *FileStore) Count() int { return len(f.files) } -// CurrentID returns the max file ID + 1 -func (f *FileStore) CurrentID() int { +// CurrentGeneration returns the max file ID + 1 +func (f *FileStore) CurrentGeneration() int { f.mu.RLock() defer f.mu.RUnlock() - return f.currentFileID + return f.currentGeneration } -// NextID returns the max file ID + 1 -func (f *FileStore) NextID() int { +// NextGeneration returns the max file ID + 1 +func (f *FileStore) NextGeneration() int { f.mu.Lock() defer f.mu.Unlock() - f.currentFileID++ - return f.currentFileID + f.currentGeneration++ + return f.currentGeneration } func (f *FileStore) Add(files ...TSMFile) { @@ -213,13 +213,13 @@ func (f *FileStore) Open() error { for _, fn := range files { // Keep track of the latest ID - id, err := f.idFromFileName(fn) + generation, _, err := f.idFromFileName(fn) if err != nil { return err } - if id >= f.currentFileID { - f.currentFileID = id + 1 + if generation >= f.currentGeneration { + f.currentGeneration = generation + 1 } file, err := os.OpenFile(fn, os.O_RDONLY, 0666) @@ -366,15 +366,21 @@ func (f *FileStore) Replace(oldFiles, newFiles []string) error { } // idFromFileName parses the segment file ID from its name -func (f *FileStore) idFromFileName(name string) (int, error) { +func (f *FileStore) idFromFileName(name string) (int, int, error) { parts := strings.Split(filepath.Base(name), ".") if len(parts) != 2 { - return 0, fmt.Errorf("file %s is named incorrectly", name) + return 0, 0, fmt.Errorf("file %s is named incorrectly", name) } - id, err := strconv.ParseUint(parts[0], 10, 32) + parts = strings.Split(parts[0], "-") + if len(parts) != 2 { + return 0, 0, fmt.Errorf("file %s is named incorrectly", name) + } - return int(id), err + generation, err := strconv.ParseUint(parts[0], 10, 32) + sequence, err := strconv.ParseUint(parts[1], 10, 32) + + return int(generation), int(sequence), err } type KeyCursor struct { diff --git a/tsdb/engine/tsm1/file_store_test.go b/tsdb/engine/tsm1/file_store_test.go index 9e447a23c7..58266e8d69 100644 --- a/tsdb/engine/tsm1/file_store_test.go +++ b/tsdb/engine/tsm1/file_store_test.go @@ -361,7 +361,7 @@ func TestFileStore_Open(t *testing.T) { t.Fatalf("file count mismatch: got %v, exp %v", got, exp) } - if got, exp := fs.CurrentID(), 4; got != exp { + if got, exp := fs.CurrentGeneration(), 4; got != exp { t.Fatalf("current ID mismatch: got %v, exp %v", got, exp) } } @@ -392,7 +392,7 @@ func TestFileStore_Remove(t *testing.T) { t.Fatalf("file count mismatch: got %v, exp %v", got, exp) } - if got, exp := fs.CurrentID(), 4; got != exp { + if got, exp := fs.CurrentGeneration(), 4; got != exp { t.Fatalf("current ID mismatch: got %v, exp %v", got, exp) } @@ -402,7 +402,7 @@ func TestFileStore_Remove(t *testing.T) { t.Fatalf("file count mismatch: got %v, exp %v", got, exp) } - if got, exp := fs.CurrentID(), 4; got != exp { + if got, exp := fs.CurrentGeneration(), 4; got != exp { t.Fatalf("current ID mismatch: got %v, exp %v", got, exp) } } @@ -563,9 +563,9 @@ func MustTempFile(dir string) *os.File { } func fatal(t *testing.T, msg string, err error) { - t.Fatalf("unexpected error %s: %v", msg, err) + t.Fatalf("unexpected error %v: %v", msg, err) } func tsmFileName(id int) string { - return fmt.Sprintf("%07d.tsm1dev", id) + return fmt.Sprintf("%09d-%09d.tsm1dev", id, 1) }