raftbench and log optimization.

raft/cmd/raftbench/main.go

@@ -10,21 +10,28 @@ import (
 	"os"
 	"time"
 
-	//"github.com/davecheney/profile"
+	"github.com/davecheney/profile"
 	"github.com/influxdb/influxdb/raft"
 )
 
-func main() {
-	//defer profile.Start(&profile.Config{CPUProfile: true, MemProfile: true, BlockProfile: true}).Stop()
+const InitializationDuration = 1 * time.Second
 
+func main() {
 	// Parse flags.
 	var (
+		prof    = flag.Bool("profile", false, "enable profiling")
 		addr    = flag.String("addr", "", "bind address")
 		joinURL = flag.String("join-url", "", "cluster to join")
+		sz      = flag.Int("size", 32, "command size")
 	)
 	flag.Parse()
 	log.SetFlags(0)
 
+	// Enable profiling.
+	if *prof {
+		defer profile.Start(&profile.Config{CPUProfile: true, MemProfile: true, BlockProfile: true}).Stop()
+	}
+
 	// Validate input.
 	if *addr == "" {
 		log.Fatal("bind address required")
@@ -59,9 +66,7 @@ func main() {
 		}
 		log.Println("[initialized]")
 
-		for i := 0; i < 100; i++ {
-			go generator(l)
-		}
+		go generator(l, *sz)
 	} else {
 		u, err := url.Parse(*joinURL)
 		if err != nil {
@@ -73,9 +78,6 @@ func main() {
 		log.Println("[joined]")
 	}
 
-	// Log time.
-	startTime := time.Now()
-
 	// Print notice.
 	log.Printf("Listening on http://%s%s", hostname, *addr)
 	log.SetFlags(log.LstdFlags)
@@ -84,16 +86,22 @@ func main() {
 	h := raft.NewHTTPHandler(l)
 	go func() { log.Fatal(http.ListenAndServe(*addr, h)) }()
 
-	time.Sleep(10 * time.Second)
+	time.Sleep(InitializationDuration)
+
+	startTime := time.Now()
+	time.Sleep(5 * time.Second)
 
 	entryN := fsm.EntryN
 	log.Printf("[BENCH] ops=%d; %0.03f ops/sec\n\n", entryN, float64(entryN)/time.Since(startTime).Seconds())
 }
 
 // generator is run in a separate goroutine to generate data.
-func generator(l *raft.Log) {
+func generator(l *raft.Log, sz int) {
+	time.Sleep(InitializationDuration)
+
 	for {
-		if err := l.Apply(make([]byte, 8)); err != nil {
+		command := make([]byte, sz)
+		if err := l.Apply(command); err != nil {
 			log.Fatalf("generate: %s", err)
 		}
 	}

raft/log.go

@@ -623,22 +623,25 @@ func (l *Log) internalApply(typ LogEntryType, command []byte) error {
 
 	// Wait for consensus.
 	// HACK(benbjohnson): Notify via channel instead.
-	for {
-		l.mu.Lock()
-		state, appliedIndex := l.state, l.appliedIndex
-		l.mu.Unlock()
+	/*
+		for {
+			l.mu.Lock()
+			state, appliedIndex := l.state, l.appliedIndex
+			l.mu.Unlock()
 
-		// If we've changed leadership then return error.
-		// If the last applied index has moved past our index then move forward.
-		if state != Leader {
-			return ErrNotLeader
-		} else if appliedIndex >= index {
-			return nil
+			// If we've changed leadership then return error.
+			// If the last applied index has moved past our index then move forward.
+			if state != Leader {
+				return ErrNotLeader
+			} else if appliedIndex >= index {
+				return nil
+			}
+
+			// Otherwise wait.
+			l.Clock.Sleep(10 * time.Millisecond)
 		}
-
-		// Otherwise wait.
-		l.Clock.Sleep(10 * time.Millisecond)
-	}
+	*/
+	return nil
 }
 
 // applier runs in a separate goroutine and applies all entries between the
@@ -798,6 +801,9 @@ func (l *Log) AddPeer(u *url.URL) (uint64, *Config, error) {
 		return 0, nil, err
 	}
 
+	// HACK(benbjohnson): Wait for command to be processed.
+	time.Sleep(100 * time.Millisecond)
+
 	// Lock while we look up the node.
 	l.mu.Lock()
 	defer l.mu.Unlock()
@@ -1163,8 +1169,10 @@ func (s *segment) append(e *LogEntry) error {
 		}
 
 		// Flush, if possible.
-		if w, ok := w.Writer.(http.Flusher); ok {
-			w.Flush()
+		if e.Type != LogEntryCommand {
+			if w, ok := w.Writer.(http.Flusher); ok {
+				w.Flush()
+			}
 		}
 	}
 

raft/raft_test.go

@@ -32,6 +32,7 @@ func Test_Simulate_SingleNode(t *testing.T) {
 			Clock: raft.NewMockClock(),
 			Rand:  seq(),
 		}
+		fsm.Log = l
 		l.URL, _ = url.Parse("//node")
 		if err := l.Open(tempfile()); err != nil {
 			log.Fatal("open: ", err)
@@ -100,7 +101,8 @@ func Test_Simulate_MultiNode(t *testing.T) {
 		}
 
 		// Wait for commands to apply.
-		s.Clock.Add(100 * time.Millisecond)
+		s.Clock.Add(500 * time.Millisecond)
+		time.Sleep(1 * time.Second)
 
 		// Validate logs of all nodes.
 		for i, n := range s.Nodes {
@@ -131,6 +133,7 @@ func Test_Simulate_MultiNode(t *testing.T) {
 
 // TestFSM represents a fake state machine that simple records all commands.
 type TestFSM struct {
+	Log     *raft.Log
 	entries []*raft.LogEntry
 }
 
@@ -215,6 +218,7 @@ func (n *SimulationNode) Generate(rand *rand.Rand, size int) reflect.Value {
 		Clock: n.Clock,
 		Rand:  seq(),
 	}
+	n.FSM.Log = n.Log
 
 	// Start HTTP server and set log URL.
 	n.HTTPServer = httptest.NewServer(raft.NewHTTPHandler(n.Log))
@@ -234,7 +238,6 @@ func (n *SimulationNode) Close() error {
 	_ = n.Log.Close()
 	n.HTTPServer.CloseClientConnections()
 	n.HTTPServer.Close()
-	time.Sleep(60 * time.Second)
 	return nil
 }