compat: Package to convert old to new config
Jessica Obermark
Jeff Wendling
parent
0d411023f2
commit
932b0bf01a
|
|
@ -0,0 +1,61 @@
|
|||
package compat
|
||||
|
||||
import (
|
||||
"github.com/influxdata/platform/storage"
|
||||
"github.com/influxdata/platform/toml"
|
||||
"github.com/influxdata/platform/tsdb/tsm1"
|
||||
)
|
||||
|
||||
// Old Config structure
|
||||
type Config struct {
|
||||
Dir string `toml:"dir"` // This doesn't exist in the new config
|
||||
|
||||
WALDir string `toml:"wal-dir"`
|
||||
WALFsyncDelay toml.Duration `toml:"wal-fsync-delay"`
|
||||
ValidateKeys bool `toml:"validate-keys"`
|
||||
|
||||
CacheMaxMemorySize toml.Size `toml:"cache-max-memory-size"`
|
||||
CacheSnapshotMemorySize toml.Size `toml:"cache-snapshot-memory-size"`
|
||||
CacheSnapshotWriteColdDuration toml.Duration `toml:"cache-snapshot-write-cold-duration"`
|
||||
CompactFullWriteColdDuration toml.Duration `toml:"compact-full-write-cold-duration"`
|
||||
CompactThroughput toml.Size `toml:"compact-throughput"`
|
||||
CompactThroughputBurst toml.Size `toml:"compact-throughput-burst"`
|
||||
|
||||
MaxConcurrentCompactions int `toml:"max-concurrent-compactions"`
|
||||
TraceLoggingEnabled bool `toml:"trace-logging-enabled"`
|
||||
TSMWillNeed bool `toml:"tsm-use-madv-willneed"`
|
||||
}
|
||||
|
||||
func Create() Config {
|
||||
return Config{
|
||||
WALDir: tsm1.DefaultWALPath,
|
||||
WALFsyncDelay: toml.Duration(tsm1.DefaultWALFsyncDelay),
|
||||
ValidateKeys: storage.DefaultValidateKeys,
|
||||
CacheMaxMemorySize: toml.Size(tsm1.DefaultCacheMaxMemorySize),
|
||||
CacheSnapshotMemorySize: toml.Size(tsm1.DefaultCacheSnapshotMemorySize),
|
||||
CacheSnapshotWriteColdDuration: toml.Duration(tsm1.DefaultCacheSnapshotWriteColdDuration),
|
||||
CompactFullWriteColdDuration: toml.Duration(tsm1.DefaultCompactFullWriteColdDuration),
|
||||
CompactThroughput: toml.Size(tsm1.DefaultCompactThroughput),
|
||||
CompactThroughputBurst: toml.Size(tsm1.DefaultCompactThroughputBurst),
|
||||
MaxConcurrentCompactions: tsm1.DefaultCompactMaxConcurrent,
|
||||
TraceLoggingEnabled: tsm1.DefaultTraceLoggingEnabled,
|
||||
TSMWillNeed: tsm1.DefaultMADVWillNeed,
|
||||
}
|
||||
}
|
||||
|
||||
func Convert(oldConfig Config) storage.Config {
|
||||
newConfig := storage.NewConfig()
|
||||
newConfig.ValidateKeys = oldConfig.ValidateKeys
|
||||
newConfig.Engine.MADVWillNeed = oldConfig.TSMWillNeed
|
||||
newConfig.Engine.TraceLoggingEnabled = oldConfig.TraceLoggingEnabled
|
||||
newConfig.Engine.Cache.MaxMemorySize = oldConfig.CacheMaxMemorySize
|
||||
newConfig.Engine.Cache.SnapshotMemorySize = oldConfig.CacheSnapshotMemorySize
|
||||
newConfig.Engine.Cache.SnapshotWriteColdDuration = oldConfig.CacheSnapshotWriteColdDuration
|
||||
newConfig.Engine.Compaction.FullWriteColdDuration = oldConfig.CompactFullWriteColdDuration
|
||||
newConfig.Engine.Compaction.Throughput = oldConfig.CompactThroughput
|
||||
newConfig.Engine.Compaction.ThroughputBurst = oldConfig.CompactThroughputBurst
|
||||
newConfig.Engine.Compaction.MaxConcurrent = oldConfig.MaxConcurrentCompactions
|
||||
newConfig.Engine.WAL.Path = oldConfig.WALDir
|
||||
newConfig.Engine.WAL.FsyncDelay = oldConfig.WALFsyncDelay
|
||||
return newConfig
|
||||
}
|
||||
|
|
@ -9,11 +9,8 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// TODO(jeff): document
|
||||
DefaultRetentionInterval = 1 * time.Hour
|
||||
|
||||
// TODO(jeff): document
|
||||
DefaultValidateKeys = false
|
||||
DefaultValidateKeys = false
|
||||
)
|
||||
|
||||
// Config holds the configuration for an Engine.
|
||||
|
|
@ -21,7 +18,7 @@ type Config struct {
|
|||
// Frequency of retention in seconds.
|
||||
RetentionInterval toml.Duration `toml:"retention-interval"`
|
||||
|
||||
// TODO(jeff): document
|
||||
// Enables unicode validation on series keys on write.
|
||||
ValidateKeys bool `toml:"validate-keys"`
|
||||
|
||||
Engine tsm1.Config `toml:"engine"`
|
||||
|
|
|
|||
|
|
@ -7,25 +7,28 @@ import (
|
|||
"github.com/influxdata/platform/toml"
|
||||
)
|
||||
|
||||
var (
|
||||
// TODO(jeff): document
|
||||
DefaultMaxConcurrentOpens = runtime.GOMAXPROCS(0)
|
||||
)
|
||||
var DefaultMaxConcurrentOpens = runtime.GOMAXPROCS(0)
|
||||
|
||||
const (
|
||||
// TODO(jeff): document
|
||||
DefaultMADVWillNeed = false
|
||||
|
||||
// TODO(jeff): document
|
||||
DefaultMADVWillNeed = false
|
||||
DefaultTraceLoggingEnabled = false
|
||||
)
|
||||
|
||||
// Config contains all of the configuration necessary to run a tsm1 engine.
|
||||
type Config struct {
|
||||
MaxConcurrentOpens int `toml:"max-concurrent-opens"`
|
||||
MADVWillNeed bool `toml:"use-madv-willneed"`
|
||||
// TODO(jeff): document
|
||||
MaxConcurrentOpens int `toml:"max-concurrent-opens"`
|
||||
|
||||
// MADVWillNeed controls whether we hint to the kernel that we intend to page
|
||||
// in mmap'd sections of TSM files. This setting defaults to off, as it has
|
||||
// been found to be problematic in some cases. It may help users who have
|
||||
// slow disks.
|
||||
MADVWillNeed bool `toml:"use-madv-willneed"`
|
||||
|
||||
// TODO(jeff): document
|
||||
TraceLoggingEnabled bool `toml:"trace-logging-enabled"`
|
||||
|
||||
// TODO(jeff): document
|
||||
FileStoreObserver FileStoreObserver `toml:"-"`
|
||||
|
||||
Compaction CompactionConfig `toml:"compaction"`
|
||||
|
|
@ -40,17 +43,17 @@ func NewConfig() Config {
|
|||
MADVWillNeed: DefaultMADVWillNeed,
|
||||
TraceLoggingEnabled: DefaultTraceLoggingEnabled,
|
||||
|
||||
Cache: CacheConfig{
|
||||
MaxMemorySize: toml.Size(DefaultCacheMaxMemorySize),
|
||||
SnapshotMemorySize: toml.Size(DefaultCacheSnapshotMemorySize),
|
||||
SnapshotWriteColdDuration: toml.Duration(DefaultCacheSnapshotWriteColdDuration),
|
||||
},
|
||||
Compaction: CompactionConfig{
|
||||
FullWriteColdDuration: toml.Duration(DefaultCompactFullWriteColdDuration),
|
||||
Throughput: toml.Size(DefaultCompactThroughput),
|
||||
ThroughputBurst: toml.Size(DefaultCompactThroughputBurst),
|
||||
MaxConcurrent: DefaultCompactMaxConcurrent,
|
||||
},
|
||||
Cache: CacheConfig{
|
||||
SnapshotMemorySize: toml.Size(DefaultCacheSnapshotMemorySize),
|
||||
MaxMemorySize: toml.Size(DefaultCacheMaxMemorySize),
|
||||
SnapshotWriteColdDuration: toml.Duration(DefaultCacheSnapshotWriteColdDuration),
|
||||
},
|
||||
WAL: WALConfig{
|
||||
Enabled: DefaultWALEnabled,
|
||||
Path: DefaultWALPath,
|
||||
|
|
@ -60,74 +63,78 @@ func NewConfig() Config {
|
|||
}
|
||||
|
||||
const (
|
||||
// DefaultCompactFullWriteColdDuration is the duration at which the engine
|
||||
// will compact all TSM files in a shard if it hasn't received a write or delete
|
||||
DefaultCompactFullWriteColdDuration = time.Duration(4 * time.Hour)
|
||||
|
||||
// DefaultCompactThroughput is the rate limit in bytes per second that we
|
||||
// will allow TSM compactions to write to disk. Not that short bursts are allowed
|
||||
// to happen at a possibly larger value, set by CompactThroughputBurst.
|
||||
// A value of 0 here will disable compaction rate limiting
|
||||
DefaultCompactThroughput = 48 * 1024 * 1024
|
||||
|
||||
// DefaultCompactThroughputBurst is the rate limit in bytes per second that we
|
||||
// will allow TSM compactions to write to disk. If this is not set, the burst value
|
||||
// will be set to equal the normal throughput
|
||||
DefaultCompactThroughputBurst = 48 * 1024 * 1024
|
||||
|
||||
// DefaultCompactMaxConcurrent is the maximum number of concurrent full and level compactions
|
||||
// that can run at one time. A value of 0 results in 50% of runtime.GOMAXPROCS(0) used at runtime.
|
||||
DefaultCompactMaxConcurrent = 0
|
||||
DefaultCompactThroughput = 48 * 1024 * 1024
|
||||
DefaultCompactThroughputBurst = 48 * 1024 * 1024
|
||||
DefaultCompactMaxConcurrent = 0
|
||||
)
|
||||
|
||||
// CompactionConfing holds all of the configuration for compactions. Eventually we want
|
||||
// to move this out of tsm1 so that it can be scheduled more intelligently.
|
||||
type CompactionConfig struct {
|
||||
// FullWriteColdDuration is the duration at which the engine will compact all TSM
|
||||
// files in a shard if it hasn't received a write or delete
|
||||
FullWriteColdDuration toml.Duration `toml:"full-write-cold-duration"`
|
||||
Throughput toml.Size `toml:"throughput"`
|
||||
ThroughputBurst toml.Size `toml:"throughput-burst"`
|
||||
MaxConcurrent int `toml:"max-concurrent"`
|
||||
|
||||
// Throughput is the rate limit in bytes per second that we will allow TSM compactions
|
||||
// to write to disk. Not that short bursts are allowed to happen at a possibly larger
|
||||
// value, set by CompactThroughputBurst. A value of 0 here will disable compaction rate
|
||||
// limiting
|
||||
Throughput toml.Size `toml:"throughput"`
|
||||
|
||||
// ThroughputBurst is the rate limit in bytes per second that we will allow TSM compactions
|
||||
// to write to disk. If this is not set, the burst value will be set to equal the normal
|
||||
// throughput
|
||||
ThroughputBurst toml.Size `toml:"throughput-burst"`
|
||||
|
||||
// MaxConcurrent is the maximum number of concurrent full and level compactions that can
|
||||
// run at one time. A value of 0 results in 50% of runtime.GOMAXPROCS(0) used at runtime.
|
||||
MaxConcurrent int `toml:"max-concurrent"`
|
||||
|
||||
// TODO(jeff): document
|
||||
PlannerCreator func(*FileStore, CompactionConfig) CompactionPlanner `toml:"-"`
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultCacheMaxMemorySize is the maximum size a shard's cache can
|
||||
// reach before it starts rejecting writes.
|
||||
DefaultCacheMaxMemorySize = 1024 * 1024 * 1024 // 1GB
|
||||
|
||||
// DefaultCacheSnapshotMemorySize is the size at which the engine will
|
||||
// snapshot the cache and write it to a TSM file, freeing up memory
|
||||
DefaultCacheSnapshotMemorySize = 25 * 1024 * 1024 // 25MB
|
||||
|
||||
// DefaultCacheSnapshotWriteColdDuration is the length of time at which
|
||||
// the engine will snapshot the cache and write it to a new TSM file if
|
||||
// the shard hasn't received writes or deletes
|
||||
DefaultCacheMaxMemorySize = 1024 * 1024 * 1024 // 1GB
|
||||
DefaultCacheSnapshotMemorySize = 25 * 1024 * 1024 // 25MB
|
||||
DefaultCacheSnapshotWriteColdDuration = time.Duration(10 * time.Minute)
|
||||
)
|
||||
|
||||
// CacheConfig holds all of the configuration for the in memory cache of values that
|
||||
// are waiting to be snapshot.
|
||||
type CacheConfig struct {
|
||||
MaxMemorySize toml.Size `toml:"max-memory-size"`
|
||||
SnapshotMemorySize toml.Size `toml:"snapshot-memory-size"`
|
||||
// MaxMemorySize is the maximum size a shard's cache can reach before it starts
|
||||
// rejecting writes.
|
||||
MaxMemorySize toml.Size `toml:"max-memory-size"`
|
||||
|
||||
// SnapshotMemorySize is the size at which the engine will snapshot the cache and
|
||||
// write it to a TSM file, freeing up memory
|
||||
SnapshotMemorySize toml.Size `toml:"snapshot-memory-size"`
|
||||
|
||||
// SnapshotWriteColdDuration is the length of time at which the engine will snapshot
|
||||
// the cache and write it to a new TSM file if the shard hasn't received writes or
|
||||
// deletes
|
||||
SnapshotWriteColdDuration toml.Duration `toml:"snapshot-write-cold-duration"`
|
||||
}
|
||||
|
||||
const (
|
||||
// TODO(jeff): document
|
||||
DefaultWALEnabled = true
|
||||
|
||||
// TODO(jeff): document
|
||||
DefaultWALPath = "../wal"
|
||||
|
||||
// TODO(jeff): document
|
||||
DefaultWALEnabled = true
|
||||
DefaultWALPath = "../wal"
|
||||
DefaultWALFsyncDelay = time.Duration(0)
|
||||
)
|
||||
|
||||
// WALConfig holds all of the configuration about the WAL.
|
||||
type WALConfig struct {
|
||||
Enabled bool `toml:"enabled"`
|
||||
Path string `toml:"path"`
|
||||
// TODO(jeff): document
|
||||
Enabled bool `toml:"enabled"`
|
||||
|
||||
// TODO(jeff): document
|
||||
Path string `toml:"path"`
|
||||
|
||||
// WALFsyncDelay is the amount of time that a write will wait before fsyncing. A
|
||||
// duration greater than 0 can be used to batch up multiple fsync calls. This is
|
||||
// useful for slower disks or when WAL write contention is seen. A value of 0 fsyncs
|
||||
// every write to the WAL.
|
||||
FsyncDelay toml.Duration `toml:"fsync-delay"`
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue