Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(storage): Small verify-wal output and test tweaks

pull/14221/head
Adam Perlin 2019-06-27 10:49:44 -07:00
parent c868ece4f6
commit f4faa9b2f5
3 changed files with 69 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/influxd/inspect/verify_wal.go
View File

@ -21,12 +21,13 @@ of entries in the scanned WAL files, in case this is of interest.
For each file, the following is output:
* The file name;
* If the file is clean ("clean" will be printed);
* The first position any corruption is found (if the file is corrupt)
* "clean" (if the file is clean) OR
The first position of any corruption that is found
In the summary section, the following is printed:
* The number of WAL files scanned;
* The number of WAL entries scanned;
* A list of files found to be corrupt`,
RunE: inspectVerifyWAL,
RunE: inspectVerifyWAL,
}
dir, err := fs.InfluxDir()
@ -34,7 +35,7 @@ In the summary section, the following is printed:
panic(err)
}
dir = filepath.Join(dir, "engine/wal")
verifyWALCommand.Flags().StringVarP(&reportTSMFlags.dataDir, "data-dir", "", dir, fmt.Sprintf("use provided data directory (defaults to %s).", dir))
verifyWALCommand.Flags().StringVarP(&verifyWALFlags.dataDir, "data-dir", "", dir, fmt.Sprintf("use provided data directory (defaults to %s).", dir))
return verifyWALCommand
}

95
storage/wal/verifier_test.go
View File

@ -3,6 +3,7 @@ package wal
import (
"context"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/influxdata/influxdb/kit/errors"
"github.com/influxdata/influxdb/tsdb/value"
"io/ioutil"
@ -13,13 +14,13 @@ import (
type Test struct {
dir string
files []string
corruptFiles []string
}
func TestVerifyWALL_CleanFile(t *testing.T) {
numTestEntries := 100
test := CreateTest(t, func() (string, []string, error) {
dir := mustCreateTempDir(t)
dir := MustTempDir()
w := NewWAL(dir)
if err := w.Open(context.Background()); err != nil {
@ -39,7 +40,7 @@ func TestVerifyWALL_CleanFile(t *testing.T) {
defer test.Close()
verifier := &Verifier{Dir: test.dir}
summary, err := verifier.Run(true)
summary, err := verifier.Run(false)
if err != nil {
t.Fatalf("Unexpected error: %v\n", err)
}
@ -57,7 +58,7 @@ func TestVerifyWALL_CleanFile(t *testing.T) {
func CreateTest(t *testing.T, createFiles func() (string, []string, error)) *Test {
t.Helper()
dir, files, err := createFiles()
dir, corruptFiles, err := createFiles()
if err != nil {
t.Fatal(err)
@ -65,14 +66,14 @@ func CreateTest(t *testing.T, createFiles func() (string, []string, error)) *Tes
return &Test{
dir: dir,
files: files,
corruptFiles: corruptFiles,
}
}
func TestVerifyWALL_CorruptFile(t *testing.T) {
test := CreateTest(t, func() (string, []string, error) {
dir := mustCreateTempDir(t)
f := mustCreateTempFile(t, dir)
dir := MustTempDir()
f := mustTempWalFile(t, dir)
writeCorruptEntries(f, t, 1)
path := f.Name()
@ -82,10 +83,9 @@ func TestVerifyWALL_CorruptFile(t *testing.T) {
defer test.Close()
verifier := &Verifier{Dir: test.dir}
expectedEntries := 1
expectedErrors := 1
expectedEntries := 2 // 1 valid entry + 1 corrupt entry
summary, err := verifier.Run(true)
summary, err := verifier.Run(false)
if err != nil {
t.Fatalf("Unexpected error when running wal verification: %v", err)
}
@ -94,17 +94,11 @@ func TestVerifyWALL_CorruptFile(t *testing.T) {
t.Fatalf("Error: expected %d entries, found %d entries", expectedEntries, summary.EntryCount)
}
if len(summary.CorruptFiles) != expectedErrors {
t.Fatalf("Error: expected %d corrupt entries, found %d corrupt entries", expectedErrors, len(summary.CorruptFiles))
}
want := test.files
want := test.corruptFiles
got := summary.CorruptFiles
t.Log("got: ", summary.CorruptFiles)
t.Log("want: ", want)
t.Log(cmp.Diff(got, want))
lessFunc := func(a, b string) bool {return a < b}
if !cmp.Equal(summary.CorruptFiles, want) {
if !cmp.Equal(summary.CorruptFiles, want, cmpopts.SortSlices(lessFunc)) {
t.Fatalf("Error: unexpected list of corrupt files %v", cmp.Diff(got, want))
}
}
@ -119,35 +113,45 @@ func writeRandomEntry(w *WAL, t *testing.T) {
}
}
func writeRandomEntryRaw(w *WALSegmentWriter, t *testing.T) {
values := map[string][]value.Value{
"cpu,host=A#!~#value": {value.NewValue(rand.Int63(), rand.Float64())},
}
entry := &WriteWALEntry{
Values: values,
}
if err := w.Write(mustMarshalEntry(entry)); err != nil {
t.Fatalf("error writing entry: %v", err)
}
}
func writeCorruptEntries(file *os.File, t *testing.T, n int) {
w := NewWALSegmentWriter(file)
// random byte sequence
corrupt := []byte{1, 255, 0, 3, 45, 26, 110}
corruption := []byte{1, 4, 0, 0, 0}
p1 := value.NewValue(1, 1.1)
values := map[string][]value.Value{
"cpu,host=A#!~#float": {p1},
}
for i := 0; i < n; i++ {
wrote, err := file.Write(corrupt)
if err != nil {
t.Fatal(err)
} else if wrote != len(corrupt) {
t.Fatal("Error writing corrupt data to file")
entry := &WriteWALEntry{
Values: values,
}
if err := w.Write(mustMarshalEntry(entry)); err != nil {
fatal(t, "write points", err)
}
if err := w.Flush(); err != nil {
fatal(t, "flush", err)
}
}
// Write some random bytes to the file to simulate corruption.
if _, err := file.Write(corruption); err != nil {
fatal(t, "corrupt WAL segment", err)
}
corrupt := []byte{1, 255, 0, 3, 45, 26, 110}
wrote, err := file.Write(corrupt)
if err != nil {
t.Fatal(err)
} else if wrote != len(corrupt) {
t.Fatal("Error writing corrupt data to file")
}
if err := file.Close(); err != nil {
t.Fatalf("Error: filed to close file: %v\n", err)
}
@ -160,16 +164,7 @@ func (t *Test) Close() {
}
}
func mustCreateTempDir(t *testing.T) string {
name, err := ioutil.TempDir(".", "wal-test")
if err != nil {
t.Fatal(err)
}
return name
}
func mustCreateTempFile(t *testing.T, dir string) *os.File {
func mustTempWalFile(t *testing.T, dir string) *os.File {
file, err := ioutil.TempFile(dir, "corrupt*.wal")
if err != nil {
t.Fatal(err)

34
storage/wal/verify.go
View File

@ -8,7 +8,6 @@ import (
"os"
"path"
"path/filepath"
"strings"
"text/tabwriter"
"time"
)
@ -57,10 +56,10 @@ func (v *Verifier) Run(print bool) (*VerificationSummary, error) {
var corruptFiles []string
var entriesScanned int
for _, file := range files {
f, err := os.OpenFile(file, os.O_RDONLY, 0600)
for _, fpath := range files {
f, err := os.OpenFile(fpath, os.O_RDONLY, 0600)
if err != nil {
fmt.Fprintf(v.Stderr, "error %s: %v. Exiting", file, err)
fmt.Fprintf(v.Stderr, "error opening file %s: %v. Exiting",fpath, err)
}
clean := true
@ -70,26 +69,31 @@ func (v *Verifier) Run(print bool) (*VerificationSummary, error) {
_, err := reader.Read()
if err != nil {
clean = false
fmt.Fprintf(tw,"%s: corrupt entry found at position %d\n", file, reader.Count())
fmt.Fprintf(tw,"%s: corrupt entry found at position %d\n", fpath, reader.Count())
corruptFiles = append(corruptFiles, fpath)
break
}
}
if clean {
fmt.Fprintf(tw, "%s: clean\n", file)
}
if !clean {
corruptFiles = append(corruptFiles, file)
fmt.Fprintf(tw, "%s: clean\n", fpath)
}
}
fmt.Fprintf(tw, "Statistics:\n")
fmt.Fprintf(tw, "Files checked: %d\n", len(files))
fmt.Fprintf(tw, "Corrupt files found: %s\n", strings.Join(corruptFiles, ","))
fmt.Fprintf(tw, "Total entries checked: %d\n", entriesScanned);
fmt.Fprintf(tw, "Time Elapsed %v\n", time.Since(start))
fmt.Fprintf(tw, "Results:\n")
fmt.Fprintf(tw, " Files checked: %d\n", len(files))
fmt.Fprintf(tw, " Total entries checked: %d\n", entriesScanned)
fmt.Fprintf(tw, " Corrupt files found: ")
if len(corruptFiles) == 0 {
fmt.Fprintf(tw,"None")
} else {
for _, name := range corruptFiles {
fmt.Fprintf(tw, "\n %s", name)
}
}
fmt.Fprintf(tw, "\nCompleted in %v\n", time.Since(start))
summary := &VerificationSummary{
EntryCount: entriesScanned,