Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

make the max leveldb open files configurable

pull/233/head
John Shahid 2014-02-05 15:02:04 -05:00
parent 7883d28e04
commit b874144cb3
8 changed files with 70 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.toml.sample
View File

@ -52,3 +52,9 @@ dir = "/tmp/influxdb/development/db"
# However, this port shouldn't be accessible from the internet.
protobuf_port = 8099
[leveldb]
# Maximum mmap open files, this will affect the virtual memory used by
# the process
max-open-files = 100

6
src/configuration/config.toml
View File

@ -49,3 +49,9 @@ seed-servers = ["hosta:8090", "hostb:8090"]
# However, this port shouldn't be accessible from the internet.
protobuf_port = 8099
[leveldb]
# Maximum mmap open files, this will affect the virtual memory used by
# the process
# max-open-files = 100

68
src/configuration/configuration.go
View File

@ -39,6 +39,10 @@ type LoggingConfig struct {
Level string
}
type LevelDbConfiguration struct {
MaxOpenFiles int `toml:"max-open-files"`
}
type TomlConfiguration struct {
Admin AdminConfig
Api ApiConfig
@ -46,25 +50,27 @@ type TomlConfiguration struct {
Storage StorageConfig
Cluster ClusterConfig
Logging LoggingConfig
LevelDb LevelDbConfiguration
Hostname string
BindAddress string `toml:"bind-address"`
}
type Configuration struct {
AdminHttpPort int
AdminAssetsDir string
ApiHttpSslPort int
ApiHttpCertPath string
ApiHttpPort int
RaftServerPort int
SeedServers []string
DataDir string
RaftDir string
ProtobufPort int
Hostname string
LogFile string
LogLevel string
BindAddress string
AdminHttpPort int
AdminAssetsDir string
ApiHttpSslPort int
ApiHttpCertPath string
ApiHttpPort int
RaftServerPort int
SeedServers []string
DataDir string
RaftDir string
ProtobufPort int
Hostname string
LogFile string
LogLevel string
BindAddress string
LevelDbMaxOpenFiles int
}
func LoadConfiguration(fileName string) *Configuration {
@ -88,20 +94,26 @@ func parseTomlConfiguration(filename string) (*Configuration, error) {
}
config := &Configuration{
AdminHttpPort: tomlConfiguration.Admin.Port,
AdminAssetsDir: tomlConfiguration.Admin.Assets,
ApiHttpPort: tomlConfiguration.Api.Port,
ApiHttpCertPath: tomlConfiguration.Api.SslCertPath,
ApiHttpSslPort: tomlConfiguration.Api.SslPort,
RaftServerPort: tomlConfiguration.Raft.Port,
RaftDir: tomlConfiguration.Raft.Dir,
ProtobufPort: tomlConfiguration.Cluster.ProtobufPort,
SeedServers: tomlConfiguration.Cluster.SeedServers,
DataDir: tomlConfiguration.Storage.Dir,
LogFile: tomlConfiguration.Logging.File,
LogLevel: tomlConfiguration.Logging.Level,
Hostname: tomlConfiguration.Hostname,
BindAddress: tomlConfiguration.BindAddress,
AdminHttpPort: tomlConfiguration.Admin.Port,
AdminAssetsDir: tomlConfiguration.Admin.Assets,
ApiHttpPort: tomlConfiguration.Api.Port,
ApiHttpCertPath: tomlConfiguration.Api.SslCertPath,
ApiHttpSslPort: tomlConfiguration.Api.SslPort,
RaftServerPort: tomlConfiguration.Raft.Port,
RaftDir: tomlConfiguration.Raft.Dir,
ProtobufPort: tomlConfiguration.Cluster.ProtobufPort,
SeedServers: tomlConfiguration.Cluster.SeedServers,
DataDir: tomlConfiguration.Storage.Dir,
LogFile: tomlConfiguration.Logging.File,
LogLevel: tomlConfiguration.Logging.Level,
Hostname: tomlConfiguration.Hostname,
BindAddress: tomlConfiguration.BindAddress,
LevelDbMaxOpenFiles: tomlConfiguration.LevelDb.MaxOpenFiles,
}
// if it wasn't set, set it to 100
if config.LevelDbMaxOpenFiles == 0 {
config.LevelDbMaxOpenFiles = 100
}
return config, nil

4
src/configuration/configuration_test.go
View File

@ -24,6 +24,10 @@ func (self *LoadConfigurationSuite) TestConfig(c *C) {
c.Assert(config.AdminAssetsDir, Equals, "./admin")
c.Assert(config.AdminHttpPort, Equals, 8083)
// the default should be 100, this shouldn't be set in the test toml
// file
c.Assert(config.LevelDbMaxOpenFiles, Equals, 100)
c.Assert(config.ApiHttpPort, Equals, 0)
c.Assert(config.ApiHttpSslPort, Equals, 8087)
c.Assert(config.ApiHttpCertPath, Equals, "../cert.pem")

2
src/coordinator/client_server_test.go
View File

@ -19,7 +19,7 @@ const DB_DIR = "/tmp/influxdb/datastore_test"
func newDatastore(c *C) datastore.Datastore {
os.MkdirAll(DB_DIR, 0744)
db, err := datastore.NewLevelDbDatastore(DB_DIR)
db, err := datastore.NewLevelDbDatastore(DB_DIR, 100)
c.Assert(err, Equals, nil)
return db
}

2
src/datastore/datastore_test.go
View File

@ -26,7 +26,7 @@ const DB_DIR = "/tmp/chronosdb/datastore_test"
func newDatastore(c *C) Datastore {
os.MkdirAll(DB_DIR, 0744)
db, err := NewLevelDbDatastore(DB_DIR)
db, err := NewLevelDbDatastore(DB_DIR, 100)
c.Assert(err, Equals, nil)
return db
}

17
src/datastore/leveldb_datastore.go
View File

@ -32,6 +32,7 @@ type LevelDbDatastore struct {
incrementLock sync.Mutex
requestId uint32
requestLogDir string
maxOpenFiles int
}
type Field struct {
@ -61,12 +62,13 @@ func getRequestLogDirForDate(baseDir string, t time.Time) string {
return filepath.Join(baseDir, logDir)
}
func NewRequestLogDb(dir string) (*requestLogDb, error) {
func NewRequestLogDb(dir string, maxOpenFiles int) (*requestLogDb, error) {
err := os.MkdirAll(dir, 0744)
if err != nil {
return nil, err
}
opts := levigo.NewOptions()
opts.SetMaxOpenFiles(maxOpenFiles)
opts.SetCache(levigo.NewLRUCache(ONE_MEGABYTE))
opts.SetCreateIfMissing(true)
opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
@ -143,7 +145,7 @@ var (
TRUE = true
)
func NewLevelDbDatastore(dbDir string) (Datastore, error) {
func NewLevelDbDatastore(dbDir string, maxOpenFiles int) (Datastore, error) {
mainDbDir := filepath.Join(dbDir, DATABASE_DIR)
requestLogDir := filepath.Join(dbDir, REQUEST_LOG_BASE_DIR)
@ -151,16 +153,17 @@ func NewLevelDbDatastore(dbDir string) (Datastore, error) {
if err != nil {
return nil, err
}
previousLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now().Add(-time.Hour*24)))
previousLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now().Add(-time.Hour*24)), maxOpenFiles)
if err != nil {
return nil, err
}
currentLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now()))
currentLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now()), maxOpenFiles)
if err != nil {
return nil, err
}
opts := levigo.NewOptions()
opts.SetMaxOpenFiles(maxOpenFiles)
opts.SetCache(levigo.NewLRUCache(ONE_GIGABYTE))
opts.SetCreateIfMissing(true)
opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
@ -195,7 +198,9 @@ func NewLevelDbDatastore(dbDir string) (Datastore, error) {
writeOptions: wo,
requestLogDir: requestLogDir,
currentRequestLog: currentLog,
previousRequestLog: previousLog}
previousRequestLog: previousLog,
maxOpenFiles: maxOpenFiles,
}
go leveldbStore.periodicallyRotateRequestLog()
@ -218,7 +223,7 @@ func (self *LevelDbDatastore) rotateRequestLog() {
oldLog := self.previousRequestLog
self.previousRequestLog = self.currentRequestLog
var err error
self.currentRequestLog, err = NewRequestLogDb(getRequestLogDirForDate(self.requestLogDir, time.Now()))
self.currentRequestLog, err = NewRequestLogDb(getRequestLogDirForDate(self.requestLogDir, time.Now()), self.maxOpenFiles)
if err != nil {
log.Error("Error creating new requst log: ", err)
panic(err)

2
src/server/server.go
View File

@ -25,7 +25,7 @@ type Server struct {
func NewServer(config *configuration.Configuration) (*Server, error) {
log.Info("Opening database at %s", config.DataDir)
db, err := datastore.NewLevelDbDatastore(config.DataDir)
db, err := datastore.NewLevelDbDatastore(config.DataDir, config.LevelDbMaxOpenFiles)
if err != nil {
return nil, err
}