From f2783353d9f4f30d719bd3a59540785b0848379f Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 23 Apr 2015 15:03:28 -0700 Subject: [PATCH 1/2] Allow Raft election timeout to be configured --- CHANGELOG.md | 1 + cmd/influxd/config.go | 15 ++++++++++++++- cmd/influxd/config_test.go | 4 +++- cmd/influxd/run.go | 5 +++++ etc/config.sample.toml | 3 +++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b28d1f29..1b1059cf45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Features - [#2398](https://github.com/influxdb/influxdb/pull/2398) Track more stats and report errors for shards. +- [#2410](https://github.com/influxdb/influxdb/pull/2410) Allow Raft election timeout to be configured. ### Bugfixes - [#2370](https://github.com/influxdb/influxdb/pull/2370): Fix data race in openTSDB endpoint. diff --git a/cmd/influxd/config.go b/cmd/influxd/config.go index c40dc701eb..5d35182d98 100644 --- a/cmd/influxd/config.go +++ b/cmd/influxd/config.go @@ -59,6 +59,9 @@ const ( // DefaultMaxTopicSize is the default maximum size in bytes a segment can consume on disk of a broker. DefaultBrokerMaxSegmentSize = 10 * 1024 * 1024 + // DefaultRaftElectionTimeout is the default Leader Election timeout. + DefaultRaftElectionTimeout = 1 * time.Second + // DefaultGraphiteDatabaseName is the default Graphite database if none is specified DefaultGraphiteDatabaseName = "graphite" @@ -108,12 +111,16 @@ var DefaultSnapshotURL = url.URL{ type Broker struct { Dir string `toml:"dir"` Enabled bool `toml:"enabled"` - Timeout Duration `toml:"election-timeout"` TruncationInterval Duration `toml:"truncation-interval"` MaxTopicSize int64 `toml:"max-topic-size"` MaxSegmentSize int64 `toml:"max-segment-size"` } +// Raft represents the Raft configuration for broker nodes. +type Raft struct { + ElectionTimeout Duration `toml:"election-timeout"` +} + // Snapshot represents the configuration for a snapshot service. Snapshot configuration // is only valid for data nodes. type Snapshot struct { @@ -178,6 +185,8 @@ type Config struct { Broker Broker `toml:"broker"` + Raft Raft `toml:"raft"` + Data Data `toml:"data"` Snapshot Snapshot `toml:"snapshot"` @@ -260,6 +269,8 @@ func NewConfig() *Config { c.Broker.MaxTopicSize = DefaultBrokerMaxTopicSize c.Broker.MaxSegmentSize = DefaultBrokerMaxSegmentSize + c.Raft.ElectionTimeout = Duration(DefaultRaftElectionTimeout) + // FIX(benbjohnson): Append where the udp servers are actually used. // config.UDPServers = append(config.UDPServers, UDPInputConfig{ // Enabled: tomlConfiguration.InputPlugins.UDPInput.Enabled, @@ -284,6 +295,8 @@ func NewTestConfig() (*Config, error) { c.Broker.Enabled = true c.Broker.Dir = filepath.Join(u.HomeDir, ".influxdb/broker") + c.Raft.ElectionTimeout = Duration(DefaultRaftElectionTimeout) + c.Data.Enabled = true c.Data.Dir = filepath.Join(u.HomeDir, ".influxdb/data") diff --git a/cmd/influxd/config_test.go b/cmd/influxd/config_test.go index 25f982e1bd..c85842bcd0 100644 --- a/cmd/influxd/config_test.go +++ b/cmd/influxd/config_test.go @@ -103,7 +103,9 @@ enabled = false # Where the broker logs are stored. The user running InfluxDB will need read/write access. dir = "/tmp/influxdb/development/broker" -# election-timeout = "2s" +# Raft distributed consensus +[raft] +election-timeout = "1s" [data] dir = "/tmp/influxdb/development/db" diff --git a/cmd/influxd/run.go b/cmd/influxd/run.go index 26d7af8748..87a86a7161 100644 --- a/cmd/influxd/run.go +++ b/cmd/influxd/run.go @@ -544,6 +544,11 @@ func (cmd *RunCommand) openBroker(brokerURLs []url.URL, h *Handler) { b.Log = l cmd.node.raftLog = l + // Create Raft clock. + clk := raft.NewClock() + clk.ElectionTimeout = time.Duration(cmd.config.Raft.ElectionTimeout) + l.Clock = clk + // Open broker so it can feed last index data to the log. if err := b.Open(path); err != nil { log.Fatalf("failed to open broker at %s : %s", path, err) diff --git a/etc/config.sample.toml b/etc/config.sample.toml index a2cebfb675..f5675f1f1d 100644 --- a/etc/config.sample.toml +++ b/etc/config.sample.toml @@ -80,6 +80,9 @@ truncation-interval = "10m" max-topic-size = 1073741824 max-segment-size = 10485760 +# Raft configuration. Controls the distributed consensus system. +election-timeout = "1s" + # Data node configuration. Data nodes are where the time-series data, in the form of # shards, is stored. [data] From 86db6df16c13416141335ff59bfb5b38fc2ef1ec Mon Sep 17 00:00:00 2001 From: Philip O'Toole Date: Thu, 23 Apr 2015 15:11:20 -0700 Subject: [PATCH 2/2] 5-second Raft election timeout during CI testing CI testing may suffer from lack of CPU resources, meaning leader elections may continually occur. This gives leader nodes more time to send out heartbeats. --- CHANGELOG.md | 2 +- cmd/influxd/config.go | 20 +++++++++++++++++++- cmd/influxd/config_test.go | 5 +++++ cmd/influxd/run.go | 3 +++ etc/config.sample.toml | 4 ++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b1059cf45..667f90de39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ### Features - [#2398](https://github.com/influxdb/influxdb/pull/2398) Track more stats and report errors for shards. -- [#2410](https://github.com/influxdb/influxdb/pull/2410) Allow Raft election timeout to be configured. +- [#2410](https://github.com/influxdb/influxdb/pull/2410) Allow Raft time configuration ### Bugfixes - [#2370](https://github.com/influxdb/influxdb/pull/2370): Fix data race in openTSDB endpoint. diff --git a/cmd/influxd/config.go b/cmd/influxd/config.go index 5d35182d98..f464cf1c5b 100644 --- a/cmd/influxd/config.go +++ b/cmd/influxd/config.go @@ -59,9 +59,18 @@ const ( // DefaultMaxTopicSize is the default maximum size in bytes a segment can consume on disk of a broker. DefaultBrokerMaxSegmentSize = 10 * 1024 * 1024 + // DefaultRaftApplyInterval is the period between applying commited Raft log entries. + DefaultRaftApplyInterval = 10 * time.Millisecond + // DefaultRaftElectionTimeout is the default Leader Election timeout. DefaultRaftElectionTimeout = 1 * time.Second + // DefaultRaftHeartbeatInterval is the interval between leader heartbeats. + DefaultRaftHeartbeatInterval = 100 * time.Millisecond + + // DefaultRaftReconnectTimeout is the default wait time between reconnections. + DefaultRaftReconnectTimeout = 10 * time.Millisecond + // DefaultGraphiteDatabaseName is the default Graphite database if none is specified DefaultGraphiteDatabaseName = "graphite" @@ -118,7 +127,10 @@ type Broker struct { // Raft represents the Raft configuration for broker nodes. type Raft struct { - ElectionTimeout Duration `toml:"election-timeout"` + ApplyInterval Duration `toml:"apply-interval"` + ElectionTimeout Duration `toml:"election-timeout"` + HeartbeatInterval Duration `toml:"heartbeat-interval"` + ReconnectTimeout Duration `toml:"reconnect-timeout"` } // Snapshot represents the configuration for a snapshot service. Snapshot configuration @@ -269,7 +281,10 @@ func NewConfig() *Config { c.Broker.MaxTopicSize = DefaultBrokerMaxTopicSize c.Broker.MaxSegmentSize = DefaultBrokerMaxSegmentSize + c.Raft.ApplyInterval = Duration(DefaultRaftApplyInterval) c.Raft.ElectionTimeout = Duration(DefaultRaftElectionTimeout) + c.Raft.HeartbeatInterval = Duration(DefaultRaftHeartbeatInterval) + c.Raft.ReconnectTimeout = Duration(DefaultRaftReconnectTimeout) // FIX(benbjohnson): Append where the udp servers are actually used. // config.UDPServers = append(config.UDPServers, UDPInputConfig{ @@ -295,7 +310,10 @@ func NewTestConfig() (*Config, error) { c.Broker.Enabled = true c.Broker.Dir = filepath.Join(u.HomeDir, ".influxdb/broker") + c.Raft.ApplyInterval = Duration(DefaultRaftApplyInterval) c.Raft.ElectionTimeout = Duration(DefaultRaftElectionTimeout) + c.Raft.HeartbeatInterval = Duration(DefaultRaftHeartbeatInterval) + c.Raft.ReconnectTimeout = Duration(DefaultRaftReconnectTimeout) c.Data.Enabled = true c.Data.Dir = filepath.Join(u.HomeDir, ".influxdb/data") diff --git a/cmd/influxd/config_test.go b/cmd/influxd/config_test.go index c85842bcd0..51351f7c70 100644 --- a/cmd/influxd/config_test.go +++ b/cmd/influxd/config_test.go @@ -105,6 +105,7 @@ dir = "/tmp/influxdb/development/broker" # Raft distributed consensus [raft] +apply-interval = "10ms" election-timeout = "1s" [data] @@ -254,6 +255,10 @@ func TestParseConfig(t *testing.T) { t.Fatalf("broker disabled mismatch: %v, got: %v", false, c.Broker.Enabled) } + if c.Raft.ApplyInterval != main.Duration(10*time.Millisecond) { + t.Fatalf("Raft apply interval mismatch: %v, got %v", 10*time.Millisecond, c.Raft.ApplyInterval) + } + if c.Data.Dir != "/tmp/influxdb/development/db" { t.Fatalf("data dir mismatch: %v", c.Data.Dir) } diff --git a/cmd/influxd/run.go b/cmd/influxd/run.go index 87a86a7161..d95fc36f40 100644 --- a/cmd/influxd/run.go +++ b/cmd/influxd/run.go @@ -546,7 +546,10 @@ func (cmd *RunCommand) openBroker(brokerURLs []url.URL, h *Handler) { // Create Raft clock. clk := raft.NewClock() + clk.ApplyInterval = time.Duration(cmd.config.Raft.ApplyInterval) clk.ElectionTimeout = time.Duration(cmd.config.Raft.ElectionTimeout) + clk.HeartbeatInterval = time.Duration(cmd.config.Raft.HeartbeatInterval) + clk.ReconnectTimeout = time.Duration(cmd.config.Raft.ReconnectTimeout) l.Clock = clk // Open broker so it can feed last index data to the log. diff --git a/etc/config.sample.toml b/etc/config.sample.toml index f5675f1f1d..8769d78bb8 100644 --- a/etc/config.sample.toml +++ b/etc/config.sample.toml @@ -81,7 +81,11 @@ max-topic-size = 1073741824 max-segment-size = 10485760 # Raft configuration. Controls the distributed consensus system. +[raft] +apply-interval = "10ms" election-timeout = "1s" +heartbeat-interval = "100ms" +reconnect-timeout = "10ms" # Data node configuration. Data nodes are where the time-series data, in the form of # shards, is stored.