From adf9830992895410a5bbe378a2804152e5fcb529 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Tue, 5 Apr 2016 13:36:47 +0100 Subject: [PATCH 1/9] Scopes influxql Protobuf package to prevent clashes Fixes #6211. In Go-land packages with the same name, e.g., internal, do not clash with each other when they're in different parts of the project. However with protobufs definitions will clash if they share the same package name. This commit renames the influxql protobuf package to `influxql` to avoid a clash with a message definition in another protobuf package called internal. Go package aliases allow us to continue to refer to the internal package as `internal` rather than `influxql`. --- influxql/ast.go | 2 +- influxql/internal/internal.pb.go | 22 +++++++++++----------- influxql/internal/internal.proto | 2 +- influxql/iterator.gen.go | 2 +- influxql/iterator.gen.go.tmpl | 4 ++-- influxql/iterator.go | 2 +- influxql/point.gen.go | 2 +- influxql/point.gen.go.tmpl | 2 +- influxql/point.go | 2 +- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/influxql/ast.go b/influxql/ast.go index 4befdbc7d9..368328fff9 100644 --- a/influxql/ast.go +++ b/influxql/ast.go @@ -11,7 +11,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/influxql/internal" + internal "github.com/influxdata/influxdb/influxql/internal" ) // DataType represents the primitive data types available in InfluxQL. diff --git a/influxql/internal/internal.pb.go b/influxql/internal/internal.pb.go index 832a131aab..cee0815b75 100644 --- a/influxql/internal/internal.pb.go +++ b/influxql/internal/internal.pb.go @@ -3,7 +3,7 @@ // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. +Package influxql is a generated protocol buffer package. It is generated from these files: internal/internal.proto @@ -19,7 +19,7 @@ It has these top-level messages: Series SeriesList */ -package internal +package influxql import proto "github.com/gogo/protobuf/proto" import fmt "fmt" @@ -471,13 +471,13 @@ func (m *SeriesList) GetItems() []*Series { } func init() { - proto.RegisterType((*Point)(nil), "internal.Point") - proto.RegisterType((*Aux)(nil), "internal.Aux") - proto.RegisterType((*IteratorOptions)(nil), "internal.IteratorOptions") - proto.RegisterType((*Measurements)(nil), "internal.Measurements") - proto.RegisterType((*Measurement)(nil), "internal.Measurement") - proto.RegisterType((*Interval)(nil), "internal.Interval") - proto.RegisterType((*IteratorStats)(nil), "internal.IteratorStats") - proto.RegisterType((*Series)(nil), "internal.Series") - proto.RegisterType((*SeriesList)(nil), "internal.SeriesList") + proto.RegisterType((*Point)(nil), "influxql.Point") + proto.RegisterType((*Aux)(nil), "influxql.Aux") + proto.RegisterType((*IteratorOptions)(nil), "influxql.IteratorOptions") + proto.RegisterType((*Measurements)(nil), "influxql.Measurements") + proto.RegisterType((*Measurement)(nil), "influxql.Measurement") + proto.RegisterType((*Interval)(nil), "influxql.Interval") + proto.RegisterType((*IteratorStats)(nil), "influxql.IteratorStats") + proto.RegisterType((*Series)(nil), "influxql.Series") + proto.RegisterType((*SeriesList)(nil), "influxql.SeriesList") } diff --git a/influxql/internal/internal.proto b/influxql/internal/internal.proto index afe42f0058..3ff58a72f3 100644 --- a/influxql/internal/internal.proto +++ b/influxql/internal/internal.proto @@ -1,4 +1,4 @@ -package internal; +package influxql; message Point { required string Name = 1; diff --git a/influxql/iterator.gen.go b/influxql/iterator.gen.go index 6d4c8fa569..94a27da5d5 100644 --- a/influxql/iterator.gen.go +++ b/influxql/iterator.gen.go @@ -18,7 +18,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/influxql/internal" + internal "github.com/influxdata/influxdb/influxql/internal" ) // DefaultStatsInterval is the default value for IteratorEncoder.StatsInterval. diff --git a/influxql/iterator.gen.go.tmpl b/influxql/iterator.gen.go.tmpl index 43b4043ba9..d0a61e8e4e 100644 --- a/influxql/iterator.gen.go.tmpl +++ b/influxql/iterator.gen.go.tmpl @@ -12,7 +12,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/influxql/internal" + internal "github.com/influxdata/influxdb/influxql/internal" ) // DefaultStatsInterval is the default value for IteratorEncoder.StatsInterval. @@ -1224,7 +1224,7 @@ func (enc *IteratorEncoder) encode{{.Name}}Iterator(itr {{.Name}}Iterator) error } default: } - + // Retrieve the next point from the iterator. p := itr.Next() diff --git a/influxql/iterator.go b/influxql/iterator.go index d27590398c..da007fbb2b 100644 --- a/influxql/iterator.go +++ b/influxql/iterator.go @@ -9,7 +9,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/influxql/internal" + internal "github.com/influxdata/influxdb/influxql/internal" ) // ErrUnknownCall is returned when operating on an unknown function call. diff --git a/influxql/point.gen.go b/influxql/point.gen.go index 0c1bafef6c..c55b0b6cce 100644 --- a/influxql/point.gen.go +++ b/influxql/point.gen.go @@ -11,7 +11,7 @@ import ( "io" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/influxql/internal" + internal "github.com/influxdata/influxdb/influxql/internal" ) // FloatPoint represents a point with a float64 value. diff --git a/influxql/point.gen.go.tmpl b/influxql/point.gen.go.tmpl index b95efca377..d8af1e869e 100644 --- a/influxql/point.gen.go.tmpl +++ b/influxql/point.gen.go.tmpl @@ -5,7 +5,7 @@ import ( "io" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/influxql/internal" + internal "github.com/influxdata/influxdb/influxql/internal" ) {{range .}} diff --git a/influxql/point.go b/influxql/point.go index 49be0d907a..8520c3659c 100644 --- a/influxql/point.go +++ b/influxql/point.go @@ -7,7 +7,7 @@ import ( "sort" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/influxql/internal" + internal "github.com/influxdata/influxdb/influxql/internal" ) // ZeroTime is the Unix nanosecond timestamp for time.Time{}. From 41acb6cef495a57db42992cc75673cb784ca8b2c Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Tue, 5 Apr 2016 13:54:11 +0100 Subject: [PATCH 2/9] Scope all internal protobuf packages --- cluster/internal/data.pb.go | 30 ++--- cluster/internal/data.proto | 2 +- cluster/rpc.go | 2 +- services/copier/internal/internal.pb.go | 9 +- services/copier/internal/internal.proto | 2 +- services/copier/service.go | 2 +- services/meta/data.go | 2 +- services/meta/internal/meta.pb.go | 150 ++++++++++++------------ services/meta/internal/meta.proto | 2 +- tsdb/internal/meta.pb.go | 11 +- tsdb/internal/meta.proto | 2 +- tsdb/meta.go | 2 +- tsdb/shard.go | 2 +- 13 files changed, 115 insertions(+), 103 deletions(-) diff --git a/cluster/internal/data.pb.go b/cluster/internal/data.pb.go index 2c80833e3d..26ce888411 100644 --- a/cluster/internal/data.pb.go +++ b/cluster/internal/data.pb.go @@ -3,7 +3,7 @@ // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. +Package cluster is a generated protocol buffer package. It is generated from these files: internal/data.proto @@ -23,7 +23,7 @@ It has these top-level messages: ExpandSourcesRequest ExpandSourcesResponse */ -package internal +package cluster import proto "github.com/gogo/protobuf/proto" import fmt "fmt" @@ -379,17 +379,17 @@ func (m *ExpandSourcesResponse) GetErr() string { } func init() { - proto.RegisterType((*WriteShardRequest)(nil), "internal.WriteShardRequest") - proto.RegisterType((*WriteShardResponse)(nil), "internal.WriteShardResponse") - proto.RegisterType((*ExecuteStatementRequest)(nil), "internal.ExecuteStatementRequest") - proto.RegisterType((*ExecuteStatementResponse)(nil), "internal.ExecuteStatementResponse") - proto.RegisterType((*CreateIteratorRequest)(nil), "internal.CreateIteratorRequest") - proto.RegisterType((*CreateIteratorResponse)(nil), "internal.CreateIteratorResponse") - proto.RegisterType((*IteratorStats)(nil), "internal.IteratorStats") - proto.RegisterType((*FieldDimensionsRequest)(nil), "internal.FieldDimensionsRequest") - proto.RegisterType((*FieldDimensionsResponse)(nil), "internal.FieldDimensionsResponse") - proto.RegisterType((*SeriesKeysRequest)(nil), "internal.SeriesKeysRequest") - proto.RegisterType((*SeriesKeysResponse)(nil), "internal.SeriesKeysResponse") - proto.RegisterType((*ExpandSourcesRequest)(nil), "internal.ExpandSourcesRequest") - proto.RegisterType((*ExpandSourcesResponse)(nil), "internal.ExpandSourcesResponse") + proto.RegisterType((*WriteShardRequest)(nil), "cluster.WriteShardRequest") + proto.RegisterType((*WriteShardResponse)(nil), "cluster.WriteShardResponse") + proto.RegisterType((*ExecuteStatementRequest)(nil), "cluster.ExecuteStatementRequest") + proto.RegisterType((*ExecuteStatementResponse)(nil), "cluster.ExecuteStatementResponse") + proto.RegisterType((*CreateIteratorRequest)(nil), "cluster.CreateIteratorRequest") + proto.RegisterType((*CreateIteratorResponse)(nil), "cluster.CreateIteratorResponse") + proto.RegisterType((*IteratorStats)(nil), "cluster.IteratorStats") + proto.RegisterType((*FieldDimensionsRequest)(nil), "cluster.FieldDimensionsRequest") + proto.RegisterType((*FieldDimensionsResponse)(nil), "cluster.FieldDimensionsResponse") + proto.RegisterType((*SeriesKeysRequest)(nil), "cluster.SeriesKeysRequest") + proto.RegisterType((*SeriesKeysResponse)(nil), "cluster.SeriesKeysResponse") + proto.RegisterType((*ExpandSourcesRequest)(nil), "cluster.ExpandSourcesRequest") + proto.RegisterType((*ExpandSourcesResponse)(nil), "cluster.ExpandSourcesResponse") } diff --git a/cluster/internal/data.proto b/cluster/internal/data.proto index f9c3c8e0ac..ed9e210f3d 100644 --- a/cluster/internal/data.proto +++ b/cluster/internal/data.proto @@ -1,4 +1,4 @@ -package internal; +package cluster; message WriteShardRequest { required uint64 ShardID = 1; diff --git a/cluster/rpc.go b/cluster/rpc.go index a06394b344..3dde860774 100644 --- a/cluster/rpc.go +++ b/cluster/rpc.go @@ -6,7 +6,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/cluster/internal" + internal "github.com/influxdata/influxdb/cluster/internal" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/influxdb/models" ) diff --git a/services/copier/internal/internal.pb.go b/services/copier/internal/internal.pb.go index 68caa62fb6..d9e7f0d135 100644 --- a/services/copier/internal/internal.pb.go +++ b/services/copier/internal/internal.pb.go @@ -3,7 +3,7 @@ // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. +Package copier is a generated protocol buffer package. It is generated from these files: internal/internal.proto @@ -12,7 +12,7 @@ It has these top-level messages: Request Response */ -package internal +package copier import proto "github.com/gogo/protobuf/proto" import fmt "fmt" @@ -54,3 +54,8 @@ func (m *Response) GetError() string { } return "" } + +func init() { + proto.RegisterType((*Request)(nil), "copier.Request") + proto.RegisterType((*Response)(nil), "copier.Response") +} diff --git a/services/copier/internal/internal.proto b/services/copier/internal/internal.proto index fb14c68124..26e5a6f198 100644 --- a/services/copier/internal/internal.proto +++ b/services/copier/internal/internal.proto @@ -1,4 +1,4 @@ -package internal; +package copier; message Request { required uint64 ShardID = 1; diff --git a/services/copier/service.go b/services/copier/service.go index cb4531e424..216ac21b63 100644 --- a/services/copier/service.go +++ b/services/copier/service.go @@ -12,7 +12,7 @@ import ( "sync" "github.com/gogo/protobuf/proto" - "github.com/influxdata/influxdb/services/copier/internal" + internal "github.com/influxdata/influxdb/services/copier/internal" "github.com/influxdata/influxdb/tcp" "github.com/influxdata/influxdb/tsdb" ) diff --git a/services/meta/data.go b/services/meta/data.go index 69ff9cd33c..6f517a1a7a 100644 --- a/services/meta/data.go +++ b/services/meta/data.go @@ -11,7 +11,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/influxql" - "github.com/influxdata/influxdb/services/meta/internal" + internal "github.com/influxdata/influxdb/services/meta/internal" ) //go:generate protoc --gogo_out=. internal/meta.proto diff --git a/services/meta/internal/meta.pb.go b/services/meta/internal/meta.pb.go index 4b28e5db58..4743e67a4c 100644 --- a/services/meta/internal/meta.pb.go +++ b/services/meta/internal/meta.pb.go @@ -3,7 +3,7 @@ // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. +Package meta is a generated protocol buffer package. It is generated from these files: internal/meta.proto @@ -52,7 +52,7 @@ It has these top-level messages: SetMetaNodeCommand DropShardCommand */ -package internal +package meta import proto "github.com/gogo/protobuf/proto" import fmt "fmt" @@ -619,7 +619,7 @@ func (m *UserPrivilege) GetPrivilege() int32 { } type Command struct { - Type *Command_Type `protobuf:"varint,1,req,name=type,enum=internal.Command_Type" json:"type,omitempty"` + Type *Command_Type `protobuf:"varint,1,req,name=type,enum=meta.Command_Type" json:"type,omitempty"` XXX_extensions map[int32]proto.Extension `json:"-"` XXX_unrecognized []byte `json:"-"` } @@ -679,7 +679,7 @@ var E_CreateNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateNodeCommand)(nil), Field: 101, - Name: "internal.CreateNodeCommand.command", + Name: "meta.CreateNodeCommand.command", Tag: "bytes,101,opt,name=command", } @@ -711,7 +711,7 @@ var E_DeleteNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DeleteNodeCommand)(nil), Field: 102, - Name: "internal.DeleteNodeCommand.command", + Name: "meta.DeleteNodeCommand.command", Tag: "bytes,102,opt,name=command", } @@ -743,7 +743,7 @@ var E_CreateDatabaseCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateDatabaseCommand)(nil), Field: 103, - Name: "internal.CreateDatabaseCommand.command", + Name: "meta.CreateDatabaseCommand.command", Tag: "bytes,103,opt,name=command", } @@ -767,7 +767,7 @@ var E_DropDatabaseCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DropDatabaseCommand)(nil), Field: 104, - Name: "internal.DropDatabaseCommand.command", + Name: "meta.DropDatabaseCommand.command", Tag: "bytes,104,opt,name=command", } @@ -799,7 +799,7 @@ var E_CreateRetentionPolicyCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateRetentionPolicyCommand)(nil), Field: 105, - Name: "internal.CreateRetentionPolicyCommand.command", + Name: "meta.CreateRetentionPolicyCommand.command", Tag: "bytes,105,opt,name=command", } @@ -831,7 +831,7 @@ var E_DropRetentionPolicyCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DropRetentionPolicyCommand)(nil), Field: 106, - Name: "internal.DropRetentionPolicyCommand.command", + Name: "meta.DropRetentionPolicyCommand.command", Tag: "bytes,106,opt,name=command", } @@ -863,7 +863,7 @@ var E_SetDefaultRetentionPolicyCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*SetDefaultRetentionPolicyCommand)(nil), Field: 107, - Name: "internal.SetDefaultRetentionPolicyCommand.command", + Name: "meta.SetDefaultRetentionPolicyCommand.command", Tag: "bytes,107,opt,name=command", } @@ -919,7 +919,7 @@ var E_UpdateRetentionPolicyCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*UpdateRetentionPolicyCommand)(nil), Field: 108, - Name: "internal.UpdateRetentionPolicyCommand.command", + Name: "meta.UpdateRetentionPolicyCommand.command", Tag: "bytes,108,opt,name=command", } @@ -959,7 +959,7 @@ var E_CreateShardGroupCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateShardGroupCommand)(nil), Field: 109, - Name: "internal.CreateShardGroupCommand.command", + Name: "meta.CreateShardGroupCommand.command", Tag: "bytes,109,opt,name=command", } @@ -999,7 +999,7 @@ var E_DeleteShardGroupCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DeleteShardGroupCommand)(nil), Field: 110, - Name: "internal.DeleteShardGroupCommand.command", + Name: "meta.DeleteShardGroupCommand.command", Tag: "bytes,110,opt,name=command", } @@ -1039,7 +1039,7 @@ var E_CreateContinuousQueryCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateContinuousQueryCommand)(nil), Field: 111, - Name: "internal.CreateContinuousQueryCommand.command", + Name: "meta.CreateContinuousQueryCommand.command", Tag: "bytes,111,opt,name=command", } @@ -1071,7 +1071,7 @@ var E_DropContinuousQueryCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DropContinuousQueryCommand)(nil), Field: 112, - Name: "internal.DropContinuousQueryCommand.command", + Name: "meta.DropContinuousQueryCommand.command", Tag: "bytes,112,opt,name=command", } @@ -1111,7 +1111,7 @@ var E_CreateUserCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateUserCommand)(nil), Field: 113, - Name: "internal.CreateUserCommand.command", + Name: "meta.CreateUserCommand.command", Tag: "bytes,113,opt,name=command", } @@ -1135,7 +1135,7 @@ var E_DropUserCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DropUserCommand)(nil), Field: 114, - Name: "internal.DropUserCommand.command", + Name: "meta.DropUserCommand.command", Tag: "bytes,114,opt,name=command", } @@ -1167,7 +1167,7 @@ var E_UpdateUserCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*UpdateUserCommand)(nil), Field: 115, - Name: "internal.UpdateUserCommand.command", + Name: "meta.UpdateUserCommand.command", Tag: "bytes,115,opt,name=command", } @@ -1207,7 +1207,7 @@ var E_SetPrivilegeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*SetPrivilegeCommand)(nil), Field: 116, - Name: "internal.SetPrivilegeCommand.command", + Name: "meta.SetPrivilegeCommand.command", Tag: "bytes,116,opt,name=command", } @@ -1231,7 +1231,7 @@ var E_SetDataCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*SetDataCommand)(nil), Field: 117, - Name: "internal.SetDataCommand.command", + Name: "meta.SetDataCommand.command", Tag: "bytes,117,opt,name=command", } @@ -1263,7 +1263,7 @@ var E_SetAdminPrivilegeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*SetAdminPrivilegeCommand)(nil), Field: 118, - Name: "internal.SetAdminPrivilegeCommand.command", + Name: "meta.SetAdminPrivilegeCommand.command", Tag: "bytes,118,opt,name=command", } @@ -1295,7 +1295,7 @@ var E_UpdateNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*UpdateNodeCommand)(nil), Field: 119, - Name: "internal.UpdateNodeCommand.command", + Name: "meta.UpdateNodeCommand.command", Tag: "bytes,119,opt,name=command", } @@ -1351,7 +1351,7 @@ var E_CreateSubscriptionCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateSubscriptionCommand)(nil), Field: 121, - Name: "internal.CreateSubscriptionCommand.command", + Name: "meta.CreateSubscriptionCommand.command", Tag: "bytes,121,opt,name=command", } @@ -1391,7 +1391,7 @@ var E_DropSubscriptionCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DropSubscriptionCommand)(nil), Field: 122, - Name: "internal.DropSubscriptionCommand.command", + Name: "meta.DropSubscriptionCommand.command", Tag: "bytes,122,opt,name=command", } @@ -1423,7 +1423,7 @@ var E_RemovePeerCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*RemovePeerCommand)(nil), Field: 123, - Name: "internal.RemovePeerCommand.command", + Name: "meta.RemovePeerCommand.command", Tag: "bytes,123,opt,name=command", } @@ -1463,7 +1463,7 @@ var E_CreateMetaNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateMetaNodeCommand)(nil), Field: 124, - Name: "internal.CreateMetaNodeCommand.command", + Name: "meta.CreateMetaNodeCommand.command", Tag: "bytes,124,opt,name=command", } @@ -1495,7 +1495,7 @@ var E_CreateDataNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*CreateDataNodeCommand)(nil), Field: 125, - Name: "internal.CreateDataNodeCommand.command", + Name: "meta.CreateDataNodeCommand.command", Tag: "bytes,125,opt,name=command", } @@ -1535,7 +1535,7 @@ var E_UpdateDataNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*UpdateDataNodeCommand)(nil), Field: 126, - Name: "internal.UpdateDataNodeCommand.command", + Name: "meta.UpdateDataNodeCommand.command", Tag: "bytes,126,opt,name=command", } @@ -1559,7 +1559,7 @@ var E_DeleteMetaNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DeleteMetaNodeCommand)(nil), Field: 127, - Name: "internal.DeleteMetaNodeCommand.command", + Name: "meta.DeleteMetaNodeCommand.command", Tag: "bytes,127,opt,name=command", } @@ -1583,7 +1583,7 @@ var E_DeleteDataNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DeleteDataNodeCommand)(nil), Field: 128, - Name: "internal.DeleteDataNodeCommand.command", + Name: "meta.DeleteDataNodeCommand.command", Tag: "bytes,128,opt,name=command", } @@ -1657,7 +1657,7 @@ var E_SetMetaNodeCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*SetMetaNodeCommand)(nil), Field: 129, - Name: "internal.SetMetaNodeCommand.command", + Name: "meta.SetMetaNodeCommand.command", Tag: "bytes,129,opt,name=command", } @@ -1681,54 +1681,54 @@ var E_DropShardCommand_Command = &proto.ExtensionDesc{ ExtendedType: (*Command)(nil), ExtensionType: (*DropShardCommand)(nil), Field: 130, - Name: "internal.DropShardCommand.command", + Name: "meta.DropShardCommand.command", Tag: "bytes,130,opt,name=command", } func init() { - proto.RegisterType((*Data)(nil), "internal.Data") - proto.RegisterType((*NodeInfo)(nil), "internal.NodeInfo") - proto.RegisterType((*DatabaseInfo)(nil), "internal.DatabaseInfo") - proto.RegisterType((*RetentionPolicyInfo)(nil), "internal.RetentionPolicyInfo") - proto.RegisterType((*ShardGroupInfo)(nil), "internal.ShardGroupInfo") - proto.RegisterType((*ShardInfo)(nil), "internal.ShardInfo") - proto.RegisterType((*SubscriptionInfo)(nil), "internal.SubscriptionInfo") - proto.RegisterType((*ShardOwner)(nil), "internal.ShardOwner") - proto.RegisterType((*ContinuousQueryInfo)(nil), "internal.ContinuousQueryInfo") - proto.RegisterType((*UserInfo)(nil), "internal.UserInfo") - proto.RegisterType((*UserPrivilege)(nil), "internal.UserPrivilege") - proto.RegisterType((*Command)(nil), "internal.Command") - proto.RegisterType((*CreateNodeCommand)(nil), "internal.CreateNodeCommand") - proto.RegisterType((*DeleteNodeCommand)(nil), "internal.DeleteNodeCommand") - proto.RegisterType((*CreateDatabaseCommand)(nil), "internal.CreateDatabaseCommand") - proto.RegisterType((*DropDatabaseCommand)(nil), "internal.DropDatabaseCommand") - proto.RegisterType((*CreateRetentionPolicyCommand)(nil), "internal.CreateRetentionPolicyCommand") - proto.RegisterType((*DropRetentionPolicyCommand)(nil), "internal.DropRetentionPolicyCommand") - proto.RegisterType((*SetDefaultRetentionPolicyCommand)(nil), "internal.SetDefaultRetentionPolicyCommand") - proto.RegisterType((*UpdateRetentionPolicyCommand)(nil), "internal.UpdateRetentionPolicyCommand") - proto.RegisterType((*CreateShardGroupCommand)(nil), "internal.CreateShardGroupCommand") - proto.RegisterType((*DeleteShardGroupCommand)(nil), "internal.DeleteShardGroupCommand") - proto.RegisterType((*CreateContinuousQueryCommand)(nil), "internal.CreateContinuousQueryCommand") - proto.RegisterType((*DropContinuousQueryCommand)(nil), "internal.DropContinuousQueryCommand") - proto.RegisterType((*CreateUserCommand)(nil), "internal.CreateUserCommand") - proto.RegisterType((*DropUserCommand)(nil), "internal.DropUserCommand") - proto.RegisterType((*UpdateUserCommand)(nil), "internal.UpdateUserCommand") - proto.RegisterType((*SetPrivilegeCommand)(nil), "internal.SetPrivilegeCommand") - proto.RegisterType((*SetDataCommand)(nil), "internal.SetDataCommand") - proto.RegisterType((*SetAdminPrivilegeCommand)(nil), "internal.SetAdminPrivilegeCommand") - proto.RegisterType((*UpdateNodeCommand)(nil), "internal.UpdateNodeCommand") - proto.RegisterType((*CreateSubscriptionCommand)(nil), "internal.CreateSubscriptionCommand") - proto.RegisterType((*DropSubscriptionCommand)(nil), "internal.DropSubscriptionCommand") - proto.RegisterType((*RemovePeerCommand)(nil), "internal.RemovePeerCommand") - proto.RegisterType((*CreateMetaNodeCommand)(nil), "internal.CreateMetaNodeCommand") - proto.RegisterType((*CreateDataNodeCommand)(nil), "internal.CreateDataNodeCommand") - proto.RegisterType((*UpdateDataNodeCommand)(nil), "internal.UpdateDataNodeCommand") - proto.RegisterType((*DeleteMetaNodeCommand)(nil), "internal.DeleteMetaNodeCommand") - proto.RegisterType((*DeleteDataNodeCommand)(nil), "internal.DeleteDataNodeCommand") - proto.RegisterType((*Response)(nil), "internal.Response") - proto.RegisterType((*SetMetaNodeCommand)(nil), "internal.SetMetaNodeCommand") - proto.RegisterType((*DropShardCommand)(nil), "internal.DropShardCommand") - proto.RegisterEnum("internal.Command_Type", Command_Type_name, Command_Type_value) + proto.RegisterType((*Data)(nil), "meta.Data") + proto.RegisterType((*NodeInfo)(nil), "meta.NodeInfo") + proto.RegisterType((*DatabaseInfo)(nil), "meta.DatabaseInfo") + proto.RegisterType((*RetentionPolicyInfo)(nil), "meta.RetentionPolicyInfo") + proto.RegisterType((*ShardGroupInfo)(nil), "meta.ShardGroupInfo") + proto.RegisterType((*ShardInfo)(nil), "meta.ShardInfo") + proto.RegisterType((*SubscriptionInfo)(nil), "meta.SubscriptionInfo") + proto.RegisterType((*ShardOwner)(nil), "meta.ShardOwner") + proto.RegisterType((*ContinuousQueryInfo)(nil), "meta.ContinuousQueryInfo") + proto.RegisterType((*UserInfo)(nil), "meta.UserInfo") + proto.RegisterType((*UserPrivilege)(nil), "meta.UserPrivilege") + proto.RegisterType((*Command)(nil), "meta.Command") + proto.RegisterType((*CreateNodeCommand)(nil), "meta.CreateNodeCommand") + proto.RegisterType((*DeleteNodeCommand)(nil), "meta.DeleteNodeCommand") + proto.RegisterType((*CreateDatabaseCommand)(nil), "meta.CreateDatabaseCommand") + proto.RegisterType((*DropDatabaseCommand)(nil), "meta.DropDatabaseCommand") + proto.RegisterType((*CreateRetentionPolicyCommand)(nil), "meta.CreateRetentionPolicyCommand") + proto.RegisterType((*DropRetentionPolicyCommand)(nil), "meta.DropRetentionPolicyCommand") + proto.RegisterType((*SetDefaultRetentionPolicyCommand)(nil), "meta.SetDefaultRetentionPolicyCommand") + proto.RegisterType((*UpdateRetentionPolicyCommand)(nil), "meta.UpdateRetentionPolicyCommand") + proto.RegisterType((*CreateShardGroupCommand)(nil), "meta.CreateShardGroupCommand") + proto.RegisterType((*DeleteShardGroupCommand)(nil), "meta.DeleteShardGroupCommand") + proto.RegisterType((*CreateContinuousQueryCommand)(nil), "meta.CreateContinuousQueryCommand") + proto.RegisterType((*DropContinuousQueryCommand)(nil), "meta.DropContinuousQueryCommand") + proto.RegisterType((*CreateUserCommand)(nil), "meta.CreateUserCommand") + proto.RegisterType((*DropUserCommand)(nil), "meta.DropUserCommand") + proto.RegisterType((*UpdateUserCommand)(nil), "meta.UpdateUserCommand") + proto.RegisterType((*SetPrivilegeCommand)(nil), "meta.SetPrivilegeCommand") + proto.RegisterType((*SetDataCommand)(nil), "meta.SetDataCommand") + proto.RegisterType((*SetAdminPrivilegeCommand)(nil), "meta.SetAdminPrivilegeCommand") + proto.RegisterType((*UpdateNodeCommand)(nil), "meta.UpdateNodeCommand") + proto.RegisterType((*CreateSubscriptionCommand)(nil), "meta.CreateSubscriptionCommand") + proto.RegisterType((*DropSubscriptionCommand)(nil), "meta.DropSubscriptionCommand") + proto.RegisterType((*RemovePeerCommand)(nil), "meta.RemovePeerCommand") + proto.RegisterType((*CreateMetaNodeCommand)(nil), "meta.CreateMetaNodeCommand") + proto.RegisterType((*CreateDataNodeCommand)(nil), "meta.CreateDataNodeCommand") + proto.RegisterType((*UpdateDataNodeCommand)(nil), "meta.UpdateDataNodeCommand") + proto.RegisterType((*DeleteMetaNodeCommand)(nil), "meta.DeleteMetaNodeCommand") + proto.RegisterType((*DeleteDataNodeCommand)(nil), "meta.DeleteDataNodeCommand") + proto.RegisterType((*Response)(nil), "meta.Response") + proto.RegisterType((*SetMetaNodeCommand)(nil), "meta.SetMetaNodeCommand") + proto.RegisterType((*DropShardCommand)(nil), "meta.DropShardCommand") + proto.RegisterEnum("meta.Command_Type", Command_Type_name, Command_Type_value) proto.RegisterExtension(E_CreateNodeCommand_Command) proto.RegisterExtension(E_DeleteNodeCommand_Command) proto.RegisterExtension(E_CreateDatabaseCommand_Command) diff --git a/services/meta/internal/meta.proto b/services/meta/internal/meta.proto index 8b7a6d6599..e220dac855 100644 --- a/services/meta/internal/meta.proto +++ b/services/meta/internal/meta.proto @@ -1,4 +1,4 @@ -package internal; +package meta; //======================================================================== // diff --git a/tsdb/internal/meta.pb.go b/tsdb/internal/meta.pb.go index c580f4dba6..c1450b39c7 100644 --- a/tsdb/internal/meta.pb.go +++ b/tsdb/internal/meta.pb.go @@ -3,7 +3,7 @@ // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. +Package meta is a generated protocol buffer package. It is generated from these files: internal/meta.proto @@ -14,7 +14,7 @@ It has these top-level messages: MeasurementFields Field */ -package internal +package meta import proto "github.com/gogo/protobuf/proto" import fmt "fmt" @@ -120,3 +120,10 @@ func (m *Field) GetType() int32 { } return 0 } + +func init() { + proto.RegisterType((*Series)(nil), "meta.Series") + proto.RegisterType((*Tag)(nil), "meta.Tag") + proto.RegisterType((*MeasurementFields)(nil), "meta.MeasurementFields") + proto.RegisterType((*Field)(nil), "meta.Field") +} diff --git a/tsdb/internal/meta.proto b/tsdb/internal/meta.proto index d1deff6c5a..c5a54f33db 100644 --- a/tsdb/internal/meta.proto +++ b/tsdb/internal/meta.proto @@ -1,4 +1,4 @@ -package internal; +package meta; //======================================================================== // diff --git a/tsdb/meta.go b/tsdb/meta.go index c01e1ab3fa..6af731ee6d 100644 --- a/tsdb/meta.go +++ b/tsdb/meta.go @@ -12,7 +12,7 @@ import ( "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/influxdb/pkg/escape" - "github.com/influxdata/influxdb/tsdb/internal" + internal "github.com/influxdata/influxdb/tsdb/internal" "github.com/gogo/protobuf/proto" ) diff --git a/tsdb/shard.go b/tsdb/shard.go index 8241fb13a0..ca3b61bb82 100644 --- a/tsdb/shard.go +++ b/tsdb/shard.go @@ -17,7 +17,7 @@ import ( "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/influxql" "github.com/influxdata/influxdb/models" - "github.com/influxdata/influxdb/tsdb/internal" + internal "github.com/influxdata/influxdb/tsdb/internal" ) const ( From b026c95192adaf1d8b515248e2d1cf1ed20a8d1a Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Wed, 6 Apr 2016 11:53:38 -0400 Subject: [PATCH 3/9] Update changelog for issue #6206 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94a9b22934..dc1d3d7e4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.13.0 [unreleased] + +### Bugfixes + +- [#6206](https://github.com/influxdata/influxdb/issues/6206): Handle nil values from the tsm1 cursor correctly. + ## v0.12.0 [2016-04-05] ### Release Notes Upgrading to this release requires a little more than just installing the new binary and starting it up. The upgrade process is very quick and should only require a minute of downtime or less. Details on [upgrading to 0.12 are here](https://docs.influxdata.com/influxdb/v0.12/administration/upgrading/). From a1256ab57aa1050ef5d419e7a4e002aa75e98dc1 Mon Sep 17 00:00:00 2001 From: aviau Date: Wed, 6 Apr 2016 17:07:12 -0400 Subject: [PATCH 4/9] include missing license for equal.go --- pkg/deep/equal.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/pkg/deep/equal.go b/pkg/deep/equal.go index 26d232022f..3fa7b4d4f5 100644 --- a/pkg/deep/equal.go +++ b/pkg/deep/equal.go @@ -1,6 +1,32 @@ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// License. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package deep // import "github.com/influxdata/influxdb/pkg/deep" From 4aeea21b79771ef14f42445caee4a788f855178e Mon Sep 17 00:00:00 2001 From: mvadu Date: Thu, 7 Apr 2016 05:24:23 +0530 Subject: [PATCH 5/9] Add AppVeyor ci config to enable ci tests on Windows Influxdb uses gdm for the dependency management. For gdm to work on Windows, it needs go 1.6. Influx still uses go 1.4.3. So go 1.6 will be sed for the pre-build stages, and go 1.4.3 to run the tests. This arrangements is working in AppVeyor now. --- CHANGELOG.md | 1 + appveyor.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 appveyor.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index dc1d3d7e4f..97c4599d54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ This release removes all of the old clustering code. It operates as a standalone - [#6153](https://github.com/influxdata/influxdb/issues/6153): Check SHARD DURATION when recreating the same database - [#6178](https://github.com/influxdata/influxdb/issues/6178): Ensure SHARD DURATION is checked when recreating a retention policy - [#6223](https://github.com/influxdata/influxdb/issues/6223): Failure to start/run on Windows. Thanks @mvadu +- [#6237](https://github.com/influxdata/influxdb/issues/6237): Enable continuous integration testing on Windows platform via AppVeyor. Thanks @mvadu ## v0.11.1 [2016-03-31] diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000000..ce50808a86 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,46 @@ +version: 0.{build} +pull_requests: + do_not_increment_build_number: true +branches: + only: + - master + +os: Windows Server 2012 R2 + +# Custom clone folder (variables are not expanded here). +clone_folder: c:\gopath\src\github.com\influxdata\influxdb + +# Environment variables +environment: + #AppVeyor has go 1.6 as default go environment + GOROOT: C:\go + GOPATH: C:\gopath + +# Scripts that run after cloning repository +install: + - set PATH=%GOROOT%\bin;%GOPATH%\bin;%PATH% + - echo %PATH% + - echo %GOPATH% + - cd C:\gopath\src\github.com\influxdata\influxdb + - go version + - go env + - go get github.com/sparrc/gdm + - cd C:\gopath\src\github.com\influxdata\influxdb + - gdm restore + +# To run your custom scripts instead of automatic MSBuild +build_script: + #Currently for gdm to work on Windows, we need go 1.6, but Influx is on go1.4.3, so switch the go environment + - set GOROOT=C:\go14 + - set PATH=%GOROOT%\bin;%GOPATH%\bin;%PATH% + - echo %PATH% + - echo %GOPATH% + - go get -t -v ./... + +# To run your custom scripts instead of automatic tests +#test_script: + - go test -race -v ./... + +# To disable deployment +deploy: off + From 4400847d6b0117836ccbf17f1caf1158bbd2e04e Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Thu, 7 Apr 2016 10:52:32 -0400 Subject: [PATCH 6/9] Throw an error when an invalid expression is used with aux iterators The following query was fixed previously: SELECT 'value' FROM cpu This ended up hitting the `buildExprIterator()` code path and was handled properly. But this query: SELECT 'value', value FROM cpu This took a different code path that would trigger a panic because it triggered a panic instead of an error condition. This code path has now been modified to trigger an error instead of a panic. Fixes #6248. --- CHANGELOG.md | 1 + influxql/select.go | 31 +++++++++++++++++++------------ influxql/select_test.go | 4 ++++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97c4599d54..d08dd9a3a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Bugfixes - [#6206](https://github.com/influxdata/influxdb/issues/6206): Handle nil values from the tsm1 cursor correctly. +- [#6248](https://github.com/influxdata/influxdb/issues/6248): Panic using incorrectly quoted "queries" field key. ## v0.12.0 [2016-04-05] ### Release Notes diff --git a/influxql/select.go b/influxql/select.go index 79b9881aac..9a050833dd 100644 --- a/influxql/select.go +++ b/influxql/select.go @@ -110,20 +110,27 @@ func buildAuxIterators(fields Fields, ic IteratorCreator, opt IteratorOptions) ( // Generate iterators for each field. itrs := make([]Iterator, len(fields)) - for i, f := range fields { - expr := Reduce(f.Expr, nil) - switch expr := expr.(type) { - case *VarRef: - itrs[i] = aitr.Iterator(expr.Val) - case *BinaryExpr: - itr, err := buildExprIterator(expr, aitr, opt) - if err != nil { - return nil, fmt.Errorf("error constructing iterator for field '%s': %s", f.String(), err) + if err := func() error { + for i, f := range fields { + expr := Reduce(f.Expr, nil) + switch expr := expr.(type) { + case *VarRef: + itrs[i] = aitr.Iterator(expr.Val) + case *BinaryExpr: + itr, err := buildExprIterator(expr, aitr, opt) + if err != nil { + return fmt.Errorf("error constructing iterator for field '%s': %s", f.String(), err) + } + itrs[i] = itr + default: + return fmt.Errorf("invalid expression type: %T", expr) } - itrs[i] = itr - default: - panic("unreachable") } + return nil + }(); err != nil { + Iterators(Iterators(itrs).filterNonNil()).Close() + aitr.Close() + return nil, err } // Background the primary iterator since there is no reader for it. diff --git a/influxql/select_test.go b/influxql/select_test.go index 18cfd020c6..f8f21cc3ea 100644 --- a/influxql/select_test.go +++ b/influxql/select_test.go @@ -2021,6 +2021,10 @@ func TestSelect_InvalidQueries(t *testing.T) { q: `SELECT 'value' FROM cpu`, err: `invalid expression type: *influxql.StringLiteral`, }, + { + q: `SELECT 'value', value FROM cpu`, + err: `invalid expression type: *influxql.StringLiteral`, + }, } for i, tt := range tests { From 1b41bec28213d63f55f5978fcefdd9ab00984f71 Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Thu, 7 Apr 2016 10:46:54 -0500 Subject: [PATCH 7/9] CreateShardGroup was incrementing meta data index even when it was idempotent --- CHANGELOG.md | 1 + services/meta/client.go | 36 ++++++++++++++++++----- services/meta/client_test.go | 55 ++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d08dd9a3a7..2dbd5b8f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [#6206](https://github.com/influxdata/influxdb/issues/6206): Handle nil values from the tsm1 cursor correctly. - [#6248](https://github.com/influxdata/influxdb/issues/6248): Panic using incorrectly quoted "queries" field key. +- [#6257](https://github.com/influxdata/influxdb/issues/6257): CreateShardGroup was incrementing meta data index even when it was idempotent. ## v0.12.0 [2016-04-05] ### Release Notes diff --git a/services/meta/client.go b/services/meta/client.go index 63cd3ed599..41c46972f5 100644 --- a/services/meta/client.go +++ b/services/meta/client.go @@ -697,6 +697,10 @@ func (c *Client) CreateShardGroup(database, policy string, timestamp time.Time) data := c.cacheData.Clone() + if sg, _ := data.ShardGroupByTimestamp(database, policy, timestamp); sg != nil { + return sg, nil + } + sgi, err := createShardGroup(data, database, policy, timestamp) if err != nil { return nil, err @@ -710,8 +714,9 @@ func (c *Client) CreateShardGroup(database, policy string, timestamp time.Time) } func createShardGroup(data *Data, database, policy string, timestamp time.Time) (*ShardGroupInfo, error) { + // It is the responsibility of the caller to check if it exists before calling this method. if sg, _ := data.ShardGroupByTimestamp(database, policy, timestamp); sg != nil { - return sg, nil + return nil, ErrShardGroupExists } if err := data.CreateShardGroup(database, policy, timestamp); err != nil { @@ -755,6 +760,7 @@ func (c *Client) PrecreateShardGroups(from, to time.Time) error { c.mu.Lock() defer c.mu.Unlock() data := c.cacheData.Clone() + var changed bool for _, di := range data.Databases { for _, rp := range di.RetentionPolicies { @@ -770,17 +776,26 @@ func (c *Client) PrecreateShardGroups(from, to time.Time) error { // Create successive shard group. nextShardGroupTime := g.EndTime.Add(1 * time.Nanosecond) - if newGroup, err := createShardGroup(data, di.Name, rp.Name, nextShardGroupTime); err != nil { - c.logger.Printf("failed to precreate successive shard group for group %d: %s", g.ID, err.Error()) - } else { - c.logger.Printf("new shard group %d successfully precreated for database %s, retention policy %s", newGroup.ID, di.Name, rp.Name) + // if it already exists, continue + if sg, _ := data.ShardGroupByTimestamp(di.Name, rp.Name, nextShardGroupTime); sg != nil { + c.logger.Printf("shard group %d exists for database %s, retention policy %s", sg.ID, di.Name, rp.Name) + continue } + newGroup, err := createShardGroup(data, di.Name, rp.Name, nextShardGroupTime) + if err != nil { + c.logger.Printf("failed to precreate successive shard group for group %d: %s", g.ID, err.Error()) + continue + } + changed = true + c.logger.Printf("new shard group %d successfully precreated for database %s, retention policy %s", newGroup.ID, di.Name, rp.Name) } } } - if err := c.commit(data); err != nil { - return err + if changed { + if err := c.commit(data); err != nil { + return err + } } return nil @@ -900,6 +915,13 @@ func (c *Client) SetData(data *Data) error { return nil } +func (c *Client) Data() Data { + c.mu.RLock() + defer c.mu.RUnlock() + d := c.cacheData.Clone() + return *d +} + // WaitForDataChanged will return a channel that will get closed when // the metastore data has changed func (c *Client) WaitForDataChanged() chan struct{} { diff --git a/services/meta/client_test.go b/services/meta/client_test.go index ef42a91edf..a8f273803f 100644 --- a/services/meta/client_test.go +++ b/services/meta/client_test.go @@ -745,6 +745,61 @@ func TestMetaClient_Shards(t *testing.T) { } } +// Tests that calling CreateShardGroup for the same time range doesn't increment the data.Index +func TestMetaClient_CreateShardGroupIdempotent(t *testing.T) { + t.Parallel() + + d, c := newClient() + defer os.RemoveAll(d) + defer c.Close() + + if _, err := c.CreateDatabase("db0"); err != nil { + t.Fatal(err) + } + + // create a shard group. + tmin := time.Now() + sg, err := c.CreateShardGroup("db0", "default", tmin) + if err != nil { + t.Fatal(err) + } else if sg == nil { + t.Fatalf("expected ShardGroup") + } + + i := c.Data().Index + t.Log("index: ", i) + + // create the same shard group. + sg, err = c.CreateShardGroup("db0", "default", tmin) + if err != nil { + t.Fatal(err) + } else if sg == nil { + t.Fatalf("expected ShardGroup") + } + + t.Log("index: ", i) + if got, exp := c.Data().Index, i; got != exp { + t.Fatalf("PrecreateShardGroups failed: invalid index, got %d, exp %d", got, exp) + } + + // make sure pre-creating is also idempotent + // Test pre-creating shard groups. + dur := sg.EndTime.Sub(sg.StartTime) + time.Nanosecond + tmax := tmin.Add(dur) + if err := c.PrecreateShardGroups(tmin, tmax); err != nil { + t.Fatal(err) + } + i = c.Data().Index + t.Log("index: ", i) + if err := c.PrecreateShardGroups(tmin, tmax); err != nil { + t.Fatal(err) + } + t.Log("index: ", i) + if got, exp := c.Data().Index, i; got != exp { + t.Fatalf("PrecreateShardGroups failed: invalid index, got %d, exp %d", got, exp) + } +} + func TestMetaClient_PersistClusterIDAfterRestart(t *testing.T) { t.Parallel() From fefd88bd81bf6352a2546420fced4751a7047627 Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Thu, 7 Apr 2016 13:47:20 -0400 Subject: [PATCH 8/9] Move changelog lines erroneously added to 0.12.0 to 0.13.0 --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dbd5b8f10..f3c5e5aeea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,15 @@ ## v0.13.0 [unreleased] +### Features + +- [#6237](https://github.com/influxdata/influxdb/issues/6237): Enable continuous integration testing on Windows platform via AppVeyor. Thanks @mvadu + ### Bugfixes - [#6206](https://github.com/influxdata/influxdb/issues/6206): Handle nil values from the tsm1 cursor correctly. - [#6248](https://github.com/influxdata/influxdb/issues/6248): Panic using incorrectly quoted "queries" field key. - [#6257](https://github.com/influxdata/influxdb/issues/6257): CreateShardGroup was incrementing meta data index even when it was idempotent. +- [#6223](https://github.com/influxdata/influxdb/issues/6223): Failure to start/run on Windows. Thanks @mvadu ## v0.12.0 [2016-04-05] ### Release Notes @@ -52,8 +57,6 @@ This release removes all of the old clustering code. It operates as a standalone - [#6152](https://github.com/influxdata/influxdb/issues/6152): Allow SHARD DURATION to be specified in isolation when creating a database - [#6153](https://github.com/influxdata/influxdb/issues/6153): Check SHARD DURATION when recreating the same database - [#6178](https://github.com/influxdata/influxdb/issues/6178): Ensure SHARD DURATION is checked when recreating a retention policy -- [#6223](https://github.com/influxdata/influxdb/issues/6223): Failure to start/run on Windows. Thanks @mvadu -- [#6237](https://github.com/influxdata/influxdb/issues/6237): Enable continuous integration testing on Windows platform via AppVeyor. Thanks @mvadu ## v0.11.1 [2016-03-31] From cea18536de939cb758e73d3a5e58238a37ed23e6 Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Wed, 6 Apr 2016 13:27:08 -0400 Subject: [PATCH 9/9] Fixing aggregate queries with no GROUP BY to include the end time Queries with a time constraint but no group by would not include the final point from the underlying iterator. Fixes #6229. --- CHANGELOG.md | 1 + influxql/iterator.gen.go | 8 ++++---- influxql/iterator.gen.go.tmpl | 2 +- influxql/iterator.go | 2 +- influxql/iterator_test.go | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3c5e5aeea..d763884b23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#6248](https://github.com/influxdata/influxdb/issues/6248): Panic using incorrectly quoted "queries" field key. - [#6257](https://github.com/influxdata/influxdb/issues/6257): CreateShardGroup was incrementing meta data index even when it was idempotent. - [#6223](https://github.com/influxdata/influxdb/issues/6223): Failure to start/run on Windows. Thanks @mvadu +- [#6229](https://github.com/influxdata/influxdb/issues/6229): Fixed aggregate queries with no GROUP BY to include the end time. ## v0.12.0 [2016-04-05] ### Release Notes diff --git a/influxql/iterator.gen.go b/influxql/iterator.gen.go index 94a27da5d5..afd861e29a 100644 --- a/influxql/iterator.gen.go +++ b/influxql/iterator.gen.go @@ -1711,7 +1711,7 @@ func (itr *floatTransformIterator) Next() *FloatPoint { // new point if possible. type floatTransformFunc func(p *FloatPoint) *FloatPoint -// floatReduceIterator executes a function to modify an existing point for every +// floatBoolTransformIterator executes a function to modify an existing point for every // output of the input iterator. type floatBoolTransformIterator struct { input FloatIterator @@ -3512,7 +3512,7 @@ func (itr *integerTransformIterator) Next() *IntegerPoint { // new point if possible. type integerTransformFunc func(p *IntegerPoint) *IntegerPoint -// integerReduceIterator executes a function to modify an existing point for every +// integerBoolTransformIterator executes a function to modify an existing point for every // output of the input iterator. type integerBoolTransformIterator struct { input IntegerIterator @@ -5313,7 +5313,7 @@ func (itr *stringTransformIterator) Next() *StringPoint { // new point if possible. type stringTransformFunc func(p *StringPoint) *StringPoint -// stringReduceIterator executes a function to modify an existing point for every +// stringBoolTransformIterator executes a function to modify an existing point for every // output of the input iterator. type stringBoolTransformIterator struct { input StringIterator @@ -7114,7 +7114,7 @@ func (itr *booleanTransformIterator) Next() *BooleanPoint { // new point if possible. type booleanTransformFunc func(p *BooleanPoint) *BooleanPoint -// booleanReduceIterator executes a function to modify an existing point for every +// booleanBoolTransformIterator executes a function to modify an existing point for every // output of the input iterator. type booleanBoolTransformIterator struct { input BooleanIterator diff --git a/influxql/iterator.gen.go.tmpl b/influxql/iterator.gen.go.tmpl index d0a61e8e4e..f7a2dbaead 100644 --- a/influxql/iterator.gen.go.tmpl +++ b/influxql/iterator.gen.go.tmpl @@ -1047,7 +1047,7 @@ func (itr *{{$k.name}}TransformIterator) Next() *{{$k.Name}}Point { // new point if possible. type {{$k.name}}TransformFunc func(p *{{$k.Name}}Point) *{{$k.Name}}Point -// {{$k.name}}ReduceIterator executes a function to modify an existing point for every +// {{$k.name}}BoolTransformIterator executes a function to modify an existing point for every // output of the input iterator. type {{$k.name}}BoolTransformIterator struct { input {{$k.Name}}Iterator diff --git a/influxql/iterator.go b/influxql/iterator.go index da007fbb2b..23a16ed1d7 100644 --- a/influxql/iterator.go +++ b/influxql/iterator.go @@ -726,7 +726,7 @@ func (opt IteratorOptions) SeekTime() int64 { // Window returns the time window [start,end) that t falls within. func (opt IteratorOptions) Window(t int64) (start, end int64) { if opt.Interval.IsZero() { - return opt.StartTime, opt.EndTime + return opt.StartTime, opt.EndTime + 1 } // Subtract the offset to the time so we calculate the correct base interval. diff --git a/influxql/iterator_test.go b/influxql/iterator_test.go index 9dfc944782..76ab2c244e 100644 --- a/influxql/iterator_test.go +++ b/influxql/iterator_test.go @@ -772,8 +772,8 @@ func TestIteratorOptions_Window_Default(t *testing.T) { if start != 0 { t.Errorf("expected start to be 0, got %d", start) } - if end != 60 { - t.Errorf("expected end to be 60, got %d", end) + if end != 61 { + t.Errorf("expected end to be 61, got %d", end) } }