From 5aee25451b4204fe1f36ff7fd067462c10cff360 Mon Sep 17 00:00:00 2001
From: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
Date: Mon, 22 Feb 2021 10:48:11 +0000
Subject: [PATCH 1/6] feat: gRPC management API definitions

---
 generated_types/build.rs                      |  27 +-
 .../storage/read}/storage_common_idpe.proto   |   4 +-
 .../iox/management/v1/base_types.proto        |  30 +++
 .../iox/management/v1/database_rules.proto    | 251 ++++++++++++++++++
 .../iox/management/v1/service.proto           |  43 +++
 .../platform/storage}/predicate.proto         |   0
 .../platform/storage}/service.proto           |   5 +-
 .../influxdata/platform/storage}/source.proto |   0
 .../platform/storage}/storage_common.proto    |   2 +-
 .../influxdata/platform/storage}/test.proto   |   0
 generated_types/{ => protos}/wal.fbs          |   0
 generated_types/src/lib.rs                    |  58 +++-
 12 files changed, 393 insertions(+), 27 deletions(-)
 rename generated_types/{ => protos/com/github/influxdata/idpe/storage/read}/storage_common_idpe.proto (87%)
 create mode 100644 generated_types/protos/influxdata/iox/management/v1/base_types.proto
 create mode 100644 generated_types/protos/influxdata/iox/management/v1/database_rules.proto
 create mode 100644 generated_types/protos/influxdata/iox/management/v1/service.proto
 rename generated_types/{ => protos/influxdata/platform/storage}/predicate.proto (100%)
 rename generated_types/{ => protos/influxdata/platform/storage}/service.proto (93%)
 rename generated_types/{ => protos/influxdata/platform/storage}/source.proto (100%)
 rename generated_types/{ => protos/influxdata/platform/storage}/storage_common.proto (99%)
 rename generated_types/{ => protos/influxdata/platform/storage}/test.proto (100%)
 rename generated_types/{ => protos}/wal.fbs (100%)

diff --git a/generated_types/build.rs b/generated_types/build.rs
index 367786400b..b58e708cd3 100644
--- a/generated_types/build.rs
+++ b/generated_types/build.rs
@@ -10,7 +10,7 @@ type Error = Box<dyn std::error::Error>;
 type Result<T, E = Error> = std::result::Result<T, E>;
 
 fn main() -> Result<()> {
-    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("protos");
 
     generate_grpc_types(&root)?;
     generate_wal_types(&root)?;
@@ -20,16 +20,25 @@ fn main() -> Result<()> {
 
 /// Schema used with IOx specific gRPC requests
 ///
-/// Creates `influxdata.platform.storage.rs` and
-/// `com.github.influxdata.idpe.storage.read.rs`
+/// Creates
+/// - `influxdata.platform.storage.rs`
+/// - `com.github.influxdata.idpe.storage.read.rs`
+/// - `influxdata.iox.management.v1.rs`
 fn generate_grpc_types(root: &Path) -> Result<()> {
+    let storage_path = root.join("influxdata/platform/storage");
+    let idpe_path = root.join("com/github/influxdata/idpe/storage/read");
+    let management_path = root.join("influxdata/iox/management/v1");
+
     let proto_files = vec![
-        root.join("test.proto"),
-        root.join("predicate.proto"),
-        root.join("storage_common.proto"),
-        root.join("storage_common_idpe.proto"),
-        root.join("service.proto"),
-        root.join("source.proto"),
+        storage_path.join("test.proto"),
+        storage_path.join("predicate.proto"),
+        storage_path.join("storage_common.proto"),
+        storage_path.join("service.proto"),
+        storage_path.join("source.proto"),
+        idpe_path.join("storage_common_idpe.proto"),
+        management_path.join("base_types.proto"),
+        management_path.join("database_rules.proto"),
+        management_path.join("service.proto"),
     ];
 
     // Tell cargo to recompile if any of these proto files are changed
diff --git a/generated_types/storage_common_idpe.proto b/generated_types/protos/com/github/influxdata/idpe/storage/read/storage_common_idpe.proto
similarity index 87%
rename from generated_types/storage_common_idpe.proto
rename to generated_types/protos/com/github/influxdata/idpe/storage/read/storage_common_idpe.proto
index e80ebe590d..da74002ded 100644
--- a/generated_types/storage_common_idpe.proto
+++ b/generated_types/protos/com/github/influxdata/idpe/storage/read/storage_common_idpe.proto
@@ -10,8 +10,8 @@ syntax = "proto3";
 package influxdata.platform.storage;
 
 import "google/protobuf/any.proto";
-import "predicate.proto";
-import "storage_common.proto";
+import "influxdata/platform/storage/predicate.proto";
+import "influxdata/platform/storage/storage_common.proto";
 
 message ReadSeriesCardinalityRequest {
   google.protobuf.Any read_series_cardinality_source = 1;
diff --git a/generated_types/protos/influxdata/iox/management/v1/base_types.proto b/generated_types/protos/influxdata/iox/management/v1/base_types.proto
new file mode 100644
index 0000000000..2618f44290
--- /dev/null
+++ b/generated_types/protos/influxdata/iox/management/v1/base_types.proto
@@ -0,0 +1,30 @@
+syntax = "proto3";
+package influxdata.iox.management.v1;
+
+enum Order {
+  ORDER_UNKNOWN = 0;
+  ORDER_ASC = 1;
+  ORDER_DESC = 2;
+}
+
+enum Aggregate {
+  AGGREGATE_UNKNOWN = 0;
+  AGGREGATE_MIN = 1;
+  AGGREGATE_MAX = 2;
+}
+
+enum ColumnType {
+  COLUMN_TYPE_UNKNOWN = 0;
+  COLUMN_TYPE_I64 = 1;
+  COLUMN_TYPE_U64 = 2;
+  COLUMN_TYPE_F64 = 3;
+  COLUMN_TYPE_STRING = 4;
+  COLUMN_TYPE_BOOL = 5;
+}
+
+message HostGroup {
+  string id = 1;
+
+  repeated string hosts = 2;
+}
+
diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
new file mode 100644
index 0000000000..4e9a780a2b
--- /dev/null
+++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
@@ -0,0 +1,251 @@
+syntax = "proto3";
+package influxdata.iox.management.v1;
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/empty.proto";
+import "influxdata/iox/management/v1/base_types.proto";
+
+// `PartitionTemplate` is used to compute the partition key of each row that
+// gets written. It can consist of the table name, a column name and its value,
+// a formatted time, or a string column and regex captures of its value. For
+// columns that do not appear in the input row, a blank value is output.
+//
+// The key is constructed in order of the template parts; thus ordering changes
+// what partition key is generated.
+message PartitionTemplate {
+  message Part {
+    message ColumnFormat {
+      string column = 1;
+      string format = 2;
+    }
+
+    oneof part {
+      google.protobuf.Empty table = 1;
+      string column = 2;
+      string time = 3;
+      ColumnFormat regex = 4;
+      ColumnFormat strfTime = 5;
+    }
+  }
+
+  repeated Part parts = 1;
+}
+
+message Matcher {
+  // Restrict selection to a specific table or table's specified by a regex
+  oneof tableMatcher {
+    string table = 1;
+    string regex = 2;
+  }
+  // A query predicate to filter rows
+  string predicate = 3;
+}
+
+message ReplicationConfig {
+  message Replication {
+    string hostGroupId = 1;
+  }
+
+  // The set of host groups that data should be replicated to. Which host a
+  // write goes to within a host group is determined by consistent hashing of
+  // the partition key. We'd use this to create a host group per
+  // availability zone, so you might have 5 availability zones with 2
+  // hosts in each. Replication will ensure that N of those zones get a
+  // write. For each zone, only a single host needs to get the write.
+  // Replication is for ensuring a write exists across multiple hosts
+  // before returning success. Its purpose is to ensure write durability,
+  // rather than write availability for query (this is covered by
+  // subscriptions).
+  repeated string replications = 1;
+
+  // The minimum number of host groups to replicate a write to before success
+  // is returned. This can be overridden on a per request basis.
+  // Replication will continue to write to the other host groups in the
+  // background.
+  uint32 replicationCount = 2;
+
+  // How long the replication queue can get before either rejecting writes or
+  // dropping missed writes. The queue is kept in memory on a
+  // per-database basis. A queue size of zero means it will only try to
+  // replicate synchronously and drop any failures.
+  uint64 replicationQueueMaxSize = 3;
+}
+
+message SubscriptionConfig {
+  message Subscription {
+    string name = 1;
+    string hostGroupId = 2;
+    Matcher matcher = 3;
+  }
+
+  // `subscriptions` are used for query servers to get data via either push
+  // or pull as it arrives. They are separate from replication as they
+  // have a different purpose. They're for query servers or other clients
+  // that want to subscribe to some subset of data being written in. This
+  // could either be specific partitions, ranges of partitions, tables, or
+  // rows matching some predicate.
+  repeated Subscription subscriptions = 1;
+}
+
+message QueryConfig {
+  // If set to `true`, this server should answer queries from one or more of
+  // of its local write buffer and any read-only partitions that it knows
+  // about. In this case, results will be merged with any others from the
+  // remote goups or read only partitions.
+  bool queryLocal = 1;
+
+  // Set `primary` to a host group if remote servers should be
+  // issued queries for this database. All hosts in the group should be
+  // queried with this server acting as the coordinator that merges
+  // results together.
+  string primary = 2;
+
+  // If a specific host in the primary group is unavailable,
+  // another host in the same position from a secondary group should be
+  // queried. For example, imagine we've partitioned the data in this DB into
+  // 4 partitions and we are replicating the data across 3 availability
+  // zones. We have 4 hosts in each of those AZs, thus they each have 1
+  // partition. We'd set the primary group to be the 4 hosts in the same
+  // AZ as this one, and the secondary groups as the hosts in the other 2 AZs.
+  repeated string secondaries = 3;
+
+  // Use `readOnlyPartitions` when a server should answer queries for
+  // partitions that come from object storage. This can be used to start
+  // up a new query server to handle queries by pointing it at a
+  // collection of partitions and then telling it to also pull
+  // data from the replication servers (writes that haven't been snapshotted
+  // into a partition).
+  repeated string readOnlyPartitions = 4;
+}
+
+message WalBufferConfig {
+  enum Rollover {
+    ROLLOVER_UNKNOWN = 0;
+
+    // Drop the old segment even though it hasn't been persisted. This part of
+    // the WAl will be lost on this server.
+    ROLLOVER_DROP_OLD_SEGMENT = 1;
+
+    // Drop the incoming write and fail silently. This favors making sure that
+    // older WAL data will be backed up.
+    ROLLOVER_DROP_INCOMING = 2;
+
+    // Reject the incoming write and return an error. The client may retry the
+    // request, which will succeed once the oldest segment has been
+    // persisted to object storage.
+    ROLLOVER_RETURN_ERROR = 3;
+  }
+
+  // The size the WAL buffer should be limited to. Once the buffer gets to
+  // this size it will drop old segments to remain below this size, but
+  // still try to hold as much in memory as possible while remaining
+  // below this threshold
+  uint64 bufferSize = 1;
+
+  // WAL segments become read only after crossing over this size. Which means
+  // that segments will always be >= this size. When old segments are
+  // dropped from of memory, at least this much space will be freed from
+  // the buffer.
+  uint64 segmentSize = 2;
+
+  // What should happen if a write comes in that would exceed the WAL buffer
+  // size and the oldest segment that could be dropped hasn't yet been
+  // persisted to object storage. If the oldest segment has been
+  // persisted, then it will be dropped from the buffer so that new writes
+  // can be accepted. This option is only for defining the behavior of what
+  // happens if that segment hasn't been persisted. If set to return an
+  // error, new writes will be rejected until the oldest segment has been
+  // persisted so that it can be cleared from memory. Alternatively, this
+  // can be set so that old segments are dropped even if they haven't been
+  // persisted. This setting is also useful for cases where persistence
+  // isn't being used and this is only for in-memory buffering.
+  Rollover bufferRollover = 3;
+
+  // If set to true, buffer segments will be written to object storage.
+  bool persistSegments = 4;
+
+  // If set, segments will be rolled over after this period of time even
+  // if they haven't hit the size threshold. This allows them to be written
+  // out to object storage as they must be immutable first.
+  google.protobuf.Duration closeSegmentAfter = 5;
+}
+
+message MutableBufferConfig {
+  message PartitionDropOrder {
+    message ColumnSort {
+      string columnName = 1;
+      ColumnType columnType = 2;
+      Aggregate columnValue = 3;
+    }
+
+    // Sort partitions by this order. Last will be dropped first.
+    Order order = 1;
+
+    // Configure sort key
+    oneof sort {
+      // The last time the partition received a write.
+      google.protobuf.Empty lastWriteTime = 2;
+
+      // When the partition was opened in the mutable buffer.
+      google.protobuf.Empty createdAtTime = 3;
+
+      // A column name, its expected type, and whether to use the min or max
+      // value. The ColumnType is necessary because the column can appear in
+      // any number of tables and be of a different type. This specifies that
+      // when sorting partitions, only columns with the given name and type
+      // should be used for the purposes of determining the partition order. If a
+      // partition doesn't have the given column in any way, the partition will
+      // appear at the beginning of the list with a null value where all
+      // partitions having null for that value will then be
+      // sorted by created_at_time desc. So if none of the partitions in the
+      // mutable buffer had this column with this type, then the partition
+      // that was created first would appear last in the list and thus be the
+      // first up to be dropped.
+      ColumnSort column = 4;
+    }
+  }
+  // The size the mutable buffer should be limited to. Once the buffer gets
+  // to this size it will drop partitions in the given order. If unable
+  // to drop partitions (because of later rules in this config) it will
+  // reject writes until it is able to drop partitions.
+  uint64 bufferSize = 1;
+
+  // If set, the mutable buffer will not drop partitions that have chunks
+  // that have not yet been persisted. Thus it will reject writes if it
+  // is over size and is unable to drop partitions. The default is to
+  // drop partitions in the sort order, regardless of whether they have
+  // unpersisted chunks or not. The WAL Buffer can be used to ensure
+  // persistence, but this may cause longer recovery times.
+  bool rejectIfNotPersisted = 2;
+
+  // Configure order to drop partitions in
+  PartitionDropOrder partitionDropOrder = 3;
+
+  // Attempt to persist partitions after they haven't received a write for
+  // this number of seconds. If not set, partitions won't be
+  // automatically persisted.
+  uint32 persistAfterColdSeconds = 4;
+}
+
+message DatabaseRules {
+  // The unencoded name of the database
+  string name = 1;
+
+  // Template that generates a partition key for each row inserted into the database
+  PartitionTemplate partitionTemplate = 2;
+
+  // Synchronous replication configuration for this database
+  ReplicationConfig replicationConfig = 3;
+
+  // Asynchronous pull-based subscription configuration for this database
+  SubscriptionConfig subscriptionConfig = 4;
+
+  // Query configuration for this database
+  QueryConfig queryConfig = 5;
+
+  // WAL configuration for this database
+  WalBufferConfig walBufferConfig = 6;
+
+  // Mutable buffer configuration for this database
+  MutableBufferConfig mutableBufferConfig = 7;
+}
\ No newline at end of file
diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto
new file mode 100644
index 0000000000..af0392f6fe
--- /dev/null
+++ b/generated_types/protos/influxdata/iox/management/v1/service.proto
@@ -0,0 +1,43 @@
+syntax = "proto3";
+package influxdata.iox.management.v1;
+
+import "google/protobuf/empty.proto";
+import "influxdata/iox/management/v1/database_rules.proto";
+
+service Management {
+  rpc Ping(google.protobuf.Empty) returns (google.protobuf.Empty);
+
+  rpc GetWriterId(google.protobuf.Empty) returns (GetWriterIdResponse);
+
+  rpc UpdateWriterId(UpdateWriterIdRequest) returns (google.protobuf.Empty);
+
+  rpc ListDatabases(google.protobuf.Empty) returns (ListDatabasesResponse);
+
+  rpc GetDatabase(GetDatabaseRequest) returns (GetDatabaseResponse);
+
+  rpc CreateDatabase(CreateDatabaseRequest) returns (google.protobuf.Empty);
+}
+
+message GetWriterIdResponse {
+  uint32 id = 1;
+}
+
+message UpdateWriterIdRequest {
+  uint32 id = 1;
+}
+
+message ListDatabasesResponse {
+  repeated string names = 1;
+}
+
+message GetDatabaseRequest {
+  string name = 1;
+}
+
+message GetDatabaseResponse {
+  DatabaseRules rules = 1;
+}
+
+message CreateDatabaseRequest {
+  DatabaseRules rules = 1;
+}
diff --git a/generated_types/predicate.proto b/generated_types/protos/influxdata/platform/storage/predicate.proto
similarity index 100%
rename from generated_types/predicate.proto
rename to generated_types/protos/influxdata/platform/storage/predicate.proto
diff --git a/generated_types/service.proto b/generated_types/protos/influxdata/platform/storage/service.proto
similarity index 93%
rename from generated_types/service.proto
rename to generated_types/protos/influxdata/platform/storage/service.proto
index 3ca768800a..3ed51d5bc5 100644
--- a/generated_types/service.proto
+++ b/generated_types/protos/influxdata/platform/storage/service.proto
@@ -8,9 +8,8 @@ syntax = "proto3";
 package influxdata.platform.storage;
 
 import "google/protobuf/empty.proto";
-import "storage_common.proto";
-import "storage_common_idpe.proto";
-
+import "influxdata/platform/storage/storage_common.proto";
+import "com/github/influxdata/idpe/storage/read/storage_common_idpe.proto";
 
 service Storage {
     // ReadFilter performs a filter operation at storage
diff --git a/generated_types/source.proto b/generated_types/protos/influxdata/platform/storage/source.proto
similarity index 100%
rename from generated_types/source.proto
rename to generated_types/protos/influxdata/platform/storage/source.proto
diff --git a/generated_types/storage_common.proto b/generated_types/protos/influxdata/platform/storage/storage_common.proto
similarity index 99%
rename from generated_types/storage_common.proto
rename to generated_types/protos/influxdata/platform/storage/storage_common.proto
index 0a1c16e83d..fce3153ec6 100644
--- a/generated_types/storage_common.proto
+++ b/generated_types/protos/influxdata/platform/storage/storage_common.proto
@@ -8,7 +8,7 @@ syntax = "proto3";
 package influxdata.platform.storage;
 
 import "google/protobuf/any.proto";
-import "predicate.proto";
+import "influxdata/platform/storage/predicate.proto";
 
 
 message ReadFilterRequest {
diff --git a/generated_types/test.proto b/generated_types/protos/influxdata/platform/storage/test.proto
similarity index 100%
rename from generated_types/test.proto
rename to generated_types/protos/influxdata/platform/storage/test.proto
diff --git a/generated_types/wal.fbs b/generated_types/protos/wal.fbs
similarity index 100%
rename from generated_types/wal.fbs
rename to generated_types/protos/wal.fbs
diff --git a/generated_types/src/lib.rs b/generated_types/src/lib.rs
index ab8649d83e..487164cad1 100644
--- a/generated_types/src/lib.rs
+++ b/generated_types/src/lib.rs
@@ -9,21 +9,55 @@
     clippy::clone_on_ref_ptr
 )]
 
-include!(concat!(env!("OUT_DIR"), "/influxdata.platform.storage.rs"));
-include!(concat!(
-    env!("OUT_DIR"),
-    "/com.github.influxdata.idpe.storage.read.rs"
-));
-include!(concat!(env!("OUT_DIR"), "/wal_generated.rs"));
+mod pb {
+    pub mod influxdata {
+        pub mod platform {
+            pub mod storage {
+                include!(concat!(env!("OUT_DIR"), "/influxdata.platform.storage.rs"));
 
-// Can't implement `Default` because `prost::Message` implements `Default`
-impl TimestampRange {
-    pub fn max() -> Self {
-        TimestampRange {
-            start: std::i64::MIN,
-            end: std::i64::MAX,
+                // Can't implement `Default` because `prost::Message` implements `Default`
+                impl TimestampRange {
+                    pub fn max() -> Self {
+                        TimestampRange {
+                            start: std::i64::MIN,
+                            end: std::i64::MAX,
+                        }
+                    }
+                }
+            }
+        }
+
+        pub mod iox {
+            pub mod management {
+                pub mod v1 {
+                    include!(concat!(env!("OUT_DIR"), "/influxdata.iox.management.v1.rs"));
+                }
+            }
+        }
+    }
+
+    pub mod com {
+        pub mod github {
+            pub mod influxdata {
+                pub mod idpe {
+                    pub mod storage {
+                        pub mod read {
+                            include!(concat!(
+                                env!("OUT_DIR"),
+                                "/com.github.influxdata.idpe.storage.read.rs"
+                            ));
+                        }
+                    }
+                }
+            }
         }
     }
 }
 
+include!(concat!(env!("OUT_DIR"), "/wal_generated.rs"));
+
+pub use pb::com::github::influxdata::idpe::storage::read::*;
+pub use pb::influxdata::platform::storage::*;
+
 pub use google_types as google;
+pub use pb::influxdata;

From 43371a344caec3042186abe09a6e492e72521d1d Mon Sep 17 00:00:00 2001
From: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
Date: Mon, 22 Feb 2021 19:38:42 +0000
Subject: [PATCH 2/6] chore: documentation fixes

---
 data_types/src/database_rules.rs                          | 6 +++---
 .../influxdata/iox/management/v1/database_rules.proto     | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs
index 14983ecc6c..7bfa430196 100644
--- a/data_types/src/database_rules.rs
+++ b/data_types/src/database_rules.rs
@@ -63,7 +63,7 @@ pub struct DatabaseRules {
     /// If set to `true`, this server should answer queries from one or more of
     /// of its local write buffer and any read-only partitions that it knows
     /// about. In this case, results will be merged with any others from the
-    /// remote goups or read only partitions.
+    /// remote goups or read-only partitions.
     #[serde(default)]
     pub query_local: bool,
     /// Set `primary_query_group` to a host group if remote servers should be
@@ -254,7 +254,7 @@ pub struct WalBufferConfig {
     /// still try to hold as much in memory as possible while remaining
     /// below this threshold
     pub buffer_size: u64,
-    /// WAL segments become read only after crossing over this size. Which means
+    /// WAL segments become read-only after crossing over this size. Which means
     /// that segments will always be >= this size. When old segments are
     /// dropped from of memory, at least this much space will be freed from
     /// the buffer.
@@ -285,7 +285,7 @@ pub struct WalBufferConfig {
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Copy)]
 pub enum WalBufferRollover {
     /// Drop the old segment even though it hasn't been persisted. This part of
-    /// the WAl will be lost on this server.
+    /// the WAL will be lost on this server.
     DropOldSegment,
     /// Drop the incoming write and fail silently. This favors making sure that
     /// older WAL data will be backed up.
diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
index 4e9a780a2b..f0d2d7d0ee 100644
--- a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
+++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
@@ -32,7 +32,7 @@ message PartitionTemplate {
 }
 
 message Matcher {
-  // Restrict selection to a specific table or table's specified by a regex
+  // Restrict selection to a specific table or tables specified by a regex
   oneof tableMatcher {
     string table = 1;
     string regex = 2;
@@ -91,7 +91,7 @@ message QueryConfig {
   // If set to `true`, this server should answer queries from one or more of
   // of its local write buffer and any read-only partitions that it knows
   // about. In this case, results will be merged with any others from the
-  // remote goups or read only partitions.
+  // remote goups or read-only partitions.
   bool queryLocal = 1;
 
   // Set `primary` to a host group if remote servers should be
@@ -123,7 +123,7 @@ message WalBufferConfig {
     ROLLOVER_UNKNOWN = 0;
 
     // Drop the old segment even though it hasn't been persisted. This part of
-    // the WAl will be lost on this server.
+    // the WAL will be lost on this server.
     ROLLOVER_DROP_OLD_SEGMENT = 1;
 
     // Drop the incoming write and fail silently. This favors making sure that
@@ -142,7 +142,7 @@ message WalBufferConfig {
   // below this threshold
   uint64 bufferSize = 1;
 
-  // WAL segments become read only after crossing over this size. Which means
+  // WAL segments become read-only after crossing over this size. Which means
   // that segments will always be >= this size. When old segments are
   // dropped from of memory, at least this much space will be freed from
   // the buffer.

From 1294d28f56ce8385758632cd4792973ff856307a Mon Sep 17 00:00:00 2001
From: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
Date: Tue, 23 Feb 2021 10:55:09 +0000
Subject: [PATCH 3/6] refactor: fix style errors

---
 generated_types/build.rs                      |  4 +-
 .../idpe/storage/read}/source.proto           |  0
 .../iox/management/v1/base_types.proto        |  7 ++-
 .../iox/management/v1/database_rules.proto    | 58 +++++++++----------
 .../iox/management/v1/service.proto           | 20 ++++---
 .../influxdata/platform/storage/service.proto |  2 +-
 .../storage}/storage_common_idpe.proto        |  0
 7 files changed, 49 insertions(+), 42 deletions(-)
 rename generated_types/protos/{influxdata/platform/storage => com/github/influxdata/idpe/storage/read}/source.proto (100%)
 rename generated_types/protos/{com/github/influxdata/idpe/storage/read => influxdata/platform/storage}/storage_common_idpe.proto (100%)

diff --git a/generated_types/build.rs b/generated_types/build.rs
index b58e708cd3..b632f70123 100644
--- a/generated_types/build.rs
+++ b/generated_types/build.rs
@@ -34,8 +34,8 @@ fn generate_grpc_types(root: &Path) -> Result<()> {
         storage_path.join("predicate.proto"),
         storage_path.join("storage_common.proto"),
         storage_path.join("service.proto"),
-        storage_path.join("source.proto"),
-        idpe_path.join("storage_common_idpe.proto"),
+        storage_path.join("storage_common_idpe.proto"),
+        idpe_path.join("source.proto"),
         management_path.join("base_types.proto"),
         management_path.join("database_rules.proto"),
         management_path.join("service.proto"),
diff --git a/generated_types/protos/influxdata/platform/storage/source.proto b/generated_types/protos/com/github/influxdata/idpe/storage/read/source.proto
similarity index 100%
rename from generated_types/protos/influxdata/platform/storage/source.proto
rename to generated_types/protos/com/github/influxdata/idpe/storage/read/source.proto
diff --git a/generated_types/protos/influxdata/iox/management/v1/base_types.proto b/generated_types/protos/influxdata/iox/management/v1/base_types.proto
index 2618f44290..0b7cd130a2 100644
--- a/generated_types/protos/influxdata/iox/management/v1/base_types.proto
+++ b/generated_types/protos/influxdata/iox/management/v1/base_types.proto
@@ -2,19 +2,19 @@ syntax = "proto3";
 package influxdata.iox.management.v1;
 
 enum Order {
-  ORDER_UNKNOWN = 0;
+  ORDER_UNSPECIFIED = 0;
   ORDER_ASC = 1;
   ORDER_DESC = 2;
 }
 
 enum Aggregate {
-  AGGREGATE_UNKNOWN = 0;
+  AGGREGATE_UNSPECIFIED = 0;
   AGGREGATE_MIN = 1;
   AGGREGATE_MAX = 2;
 }
 
 enum ColumnType {
-  COLUMN_TYPE_UNKNOWN = 0;
+  COLUMN_TYPE_UNSPECIFIED = 0;
   COLUMN_TYPE_I64 = 1;
   COLUMN_TYPE_U64 = 2;
   COLUMN_TYPE_F64 = 3;
@@ -25,6 +25,7 @@ enum ColumnType {
 message HostGroup {
   string id = 1;
 
+  // connection strings for remote hosts.
   repeated string hosts = 2;
 }
 
diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
index f0d2d7d0ee..e09f3e18f7 100644
--- a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
+++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
@@ -24,7 +24,7 @@ message PartitionTemplate {
       string column = 2;
       string time = 3;
       ColumnFormat regex = 4;
-      ColumnFormat strfTime = 5;
+      ColumnFormat strf_time = 5;
     }
   }
 
@@ -33,7 +33,7 @@ message PartitionTemplate {
 
 message Matcher {
   // Restrict selection to a specific table or tables specified by a regex
-  oneof tableMatcher {
+  oneof table_matcher {
     string table = 1;
     string regex = 2;
   }
@@ -43,7 +43,7 @@ message Matcher {
 
 message ReplicationConfig {
   message Replication {
-    string hostGroupId = 1;
+    string host_group_id = 1;
   }
 
   // The set of host groups that data should be replicated to. Which host a
@@ -62,19 +62,19 @@ message ReplicationConfig {
   // is returned. This can be overridden on a per request basis.
   // Replication will continue to write to the other host groups in the
   // background.
-  uint32 replicationCount = 2;
+  uint32 replication_count = 2;
 
   // How long the replication queue can get before either rejecting writes or
   // dropping missed writes. The queue is kept in memory on a
   // per-database basis. A queue size of zero means it will only try to
   // replicate synchronously and drop any failures.
-  uint64 replicationQueueMaxSize = 3;
+  uint64 replication_queue_max_size = 3;
 }
 
 message SubscriptionConfig {
   message Subscription {
     string name = 1;
-    string hostGroupId = 2;
+    string host_group_id = 2;
     Matcher matcher = 3;
   }
 
@@ -92,7 +92,7 @@ message QueryConfig {
   // of its local write buffer and any read-only partitions that it knows
   // about. In this case, results will be merged with any others from the
   // remote goups or read-only partitions.
-  bool queryLocal = 1;
+  bool query_local = 1;
 
   // Set `primary` to a host group if remote servers should be
   // issued queries for this database. All hosts in the group should be
@@ -115,12 +115,12 @@ message QueryConfig {
   // collection of partitions and then telling it to also pull
   // data from the replication servers (writes that haven't been snapshotted
   // into a partition).
-  repeated string readOnlyPartitions = 4;
+  repeated string read_only_partitions = 4;
 }
 
 message WalBufferConfig {
   enum Rollover {
-    ROLLOVER_UNKNOWN = 0;
+    ROLLOVER_UNSPECIFIED = 0;
 
     // Drop the old segment even though it hasn't been persisted. This part of
     // the WAL will be lost on this server.
@@ -140,13 +140,13 @@ message WalBufferConfig {
   // this size it will drop old segments to remain below this size, but
   // still try to hold as much in memory as possible while remaining
   // below this threshold
-  uint64 bufferSize = 1;
+  uint64 buffer_size = 1;
 
   // WAL segments become read-only after crossing over this size. Which means
   // that segments will always be >= this size. When old segments are
   // dropped from of memory, at least this much space will be freed from
   // the buffer.
-  uint64 segmentSize = 2;
+  uint64 segment_size = 2;
 
   // What should happen if a write comes in that would exceed the WAL buffer
   // size and the oldest segment that could be dropped hasn't yet been
@@ -159,23 +159,23 @@ message WalBufferConfig {
   // can be set so that old segments are dropped even if they haven't been
   // persisted. This setting is also useful for cases where persistence
   // isn't being used and this is only for in-memory buffering.
-  Rollover bufferRollover = 3;
+  Rollover buffer_rollover = 3;
 
   // If set to true, buffer segments will be written to object storage.
-  bool persistSegments = 4;
+  bool persist_segments = 4;
 
   // If set, segments will be rolled over after this period of time even
   // if they haven't hit the size threshold. This allows them to be written
   // out to object storage as they must be immutable first.
-  google.protobuf.Duration closeSegmentAfter = 5;
+  google.protobuf.Duration close_segment_after = 5;
 }
 
 message MutableBufferConfig {
   message PartitionDropOrder {
     message ColumnSort {
-      string columnName = 1;
-      ColumnType columnType = 2;
-      Aggregate columnValue = 3;
+      string column_name = 1;
+      ColumnType column_type = 2;
+      Aggregate column_value = 3;
     }
 
     // Sort partitions by this order. Last will be dropped first.
@@ -184,10 +184,10 @@ message MutableBufferConfig {
     // Configure sort key
     oneof sort {
       // The last time the partition received a write.
-      google.protobuf.Empty lastWriteTime = 2;
+      google.protobuf.Empty last_write_time = 2;
 
       // When the partition was opened in the mutable buffer.
-      google.protobuf.Empty createdAtTime = 3;
+      google.protobuf.Empty created_at_time = 3;
 
       // A column name, its expected type, and whether to use the min or max
       // value. The ColumnType is necessary because the column can appear in
@@ -208,7 +208,7 @@ message MutableBufferConfig {
   // to this size it will drop partitions in the given order. If unable
   // to drop partitions (because of later rules in this config) it will
   // reject writes until it is able to drop partitions.
-  uint64 bufferSize = 1;
+  uint64 buffer_size = 1;
 
   // If set, the mutable buffer will not drop partitions that have chunks
   // that have not yet been persisted. Thus it will reject writes if it
@@ -216,15 +216,15 @@ message MutableBufferConfig {
   // drop partitions in the sort order, regardless of whether they have
   // unpersisted chunks or not. The WAL Buffer can be used to ensure
   // persistence, but this may cause longer recovery times.
-  bool rejectIfNotPersisted = 2;
+  bool reject_if_not_persisted = 2;
 
   // Configure order to drop partitions in
-  PartitionDropOrder partitionDropOrder = 3;
+  PartitionDropOrder partition_drop_order = 3;
 
   // Attempt to persist partitions after they haven't received a write for
   // this number of seconds. If not set, partitions won't be
   // automatically persisted.
-  uint32 persistAfterColdSeconds = 4;
+  uint32 persist_after_cold_seconds = 4;
 }
 
 message DatabaseRules {
@@ -232,20 +232,20 @@ message DatabaseRules {
   string name = 1;
 
   // Template that generates a partition key for each row inserted into the database
-  PartitionTemplate partitionTemplate = 2;
+  PartitionTemplate partition_template = 2;
 
   // Synchronous replication configuration for this database
-  ReplicationConfig replicationConfig = 3;
+  ReplicationConfig replication_config = 3;
 
   // Asynchronous pull-based subscription configuration for this database
-  SubscriptionConfig subscriptionConfig = 4;
+  SubscriptionConfig subscription_config = 4;
 
   // Query configuration for this database
-  QueryConfig queryConfig = 5;
+  QueryConfig query_config = 5;
 
   // WAL configuration for this database
-  WalBufferConfig walBufferConfig = 6;
+  WalBufferConfig wal_buffer_config = 6;
 
   // Mutable buffer configuration for this database
-  MutableBufferConfig mutableBufferConfig = 7;
+  MutableBufferConfig mutable_buffer_config = 7;
 }
\ No newline at end of file
diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto
index af0392f6fe..73735431c1 100644
--- a/generated_types/protos/influxdata/iox/management/v1/service.proto
+++ b/generated_types/protos/influxdata/iox/management/v1/service.proto
@@ -4,20 +4,20 @@ package influxdata.iox.management.v1;
 import "google/protobuf/empty.proto";
 import "influxdata/iox/management/v1/database_rules.proto";
 
-service Management {
-  rpc Ping(google.protobuf.Empty) returns (google.protobuf.Empty);
+service ManagementService {
+  rpc GetWriterId(GetWriterIdRequest) returns (GetWriterIdResponse);
 
-  rpc GetWriterId(google.protobuf.Empty) returns (GetWriterIdResponse);
+  rpc UpdateWriterId(UpdateWriterIdRequest) returns (UpdateWriterIdResponse);
 
-  rpc UpdateWriterId(UpdateWriterIdRequest) returns (google.protobuf.Empty);
-
-  rpc ListDatabases(google.protobuf.Empty) returns (ListDatabasesResponse);
+  rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse);
 
   rpc GetDatabase(GetDatabaseRequest) returns (GetDatabaseResponse);
 
-  rpc CreateDatabase(CreateDatabaseRequest) returns (google.protobuf.Empty);
+  rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse);
 }
 
+message GetWriterIdRequest {}
+
 message GetWriterIdResponse {
   uint32 id = 1;
 }
@@ -26,6 +26,10 @@ message UpdateWriterIdRequest {
   uint32 id = 1;
 }
 
+message UpdateWriterIdResponse {}
+
+message ListDatabasesRequest {}
+
 message ListDatabasesResponse {
   repeated string names = 1;
 }
@@ -41,3 +45,5 @@ message GetDatabaseResponse {
 message CreateDatabaseRequest {
   DatabaseRules rules = 1;
 }
+
+message CreateDatabaseResponse {}
diff --git a/generated_types/protos/influxdata/platform/storage/service.proto b/generated_types/protos/influxdata/platform/storage/service.proto
index 3ed51d5bc5..9d2e78206c 100644
--- a/generated_types/protos/influxdata/platform/storage/service.proto
+++ b/generated_types/protos/influxdata/platform/storage/service.proto
@@ -9,7 +9,7 @@ package influxdata.platform.storage;
 
 import "google/protobuf/empty.proto";
 import "influxdata/platform/storage/storage_common.proto";
-import "com/github/influxdata/idpe/storage/read/storage_common_idpe.proto";
+import "influxdata/platform/storage/storage_common_idpe.proto";
 
 service Storage {
     // ReadFilter performs a filter operation at storage
diff --git a/generated_types/protos/com/github/influxdata/idpe/storage/read/storage_common_idpe.proto b/generated_types/protos/influxdata/platform/storage/storage_common_idpe.proto
similarity index 100%
rename from generated_types/protos/com/github/influxdata/idpe/storage/read/storage_common_idpe.proto
rename to generated_types/protos/influxdata/platform/storage/storage_common_idpe.proto

From 0fc711c9dfb17d916c159dc90ba23cc80a172ef5 Mon Sep 17 00:00:00 2001
From: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
Date: Tue, 23 Feb 2021 16:36:11 +0000
Subject: [PATCH 4/6] fix: don't use default Matcher.table_matcher "variant"

---
 .../influxdata/iox/management/v1/base_types.proto     |  1 -
 .../influxdata/iox/management/v1/database_rules.proto | 11 ++++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/generated_types/protos/influxdata/iox/management/v1/base_types.proto b/generated_types/protos/influxdata/iox/management/v1/base_types.proto
index 0b7cd130a2..90398b778a 100644
--- a/generated_types/protos/influxdata/iox/management/v1/base_types.proto
+++ b/generated_types/protos/influxdata/iox/management/v1/base_types.proto
@@ -28,4 +28,3 @@ message HostGroup {
   // connection strings for remote hosts.
   repeated string hosts = 2;
 }
-
diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
index e09f3e18f7..5d669419a0 100644
--- a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
+++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
@@ -32,13 +32,14 @@ message PartitionTemplate {
 }
 
 message Matcher {
+  // A query predicate to filter rows
+  string predicate = 1;
   // Restrict selection to a specific table or tables specified by a regex
   oneof table_matcher {
-    string table = 1;
-    string regex = 2;
+    google.protobuf.Empty all = 2;
+    string table = 3;
+    string regex = 4;
   }
-  // A query predicate to filter rows
-  string predicate = 3;
 }
 
 message ReplicationConfig {
@@ -248,4 +249,4 @@ message DatabaseRules {
 
   // Mutable buffer configuration for this database
   MutableBufferConfig mutable_buffer_config = 7;
-}
\ No newline at end of file
+}

From a7a77d9cd737f0e12c47050846adcbfc5a10725d Mon Sep 17 00:00:00 2001
From: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
Date: Tue, 23 Feb 2021 17:02:15 +0000
Subject: [PATCH 5/6] refactor: remove unused Replication message

---
 .../protos/influxdata/iox/management/v1/database_rules.proto  | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
index 5d669419a0..df953f4224 100644
--- a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
+++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto
@@ -43,10 +43,6 @@ message Matcher {
 }
 
 message ReplicationConfig {
-  message Replication {
-    string host_group_id = 1;
-  }
-
   // The set of host groups that data should be replicated to. Which host a
   // write goes to within a host group is determined by consistent hashing of
   // the partition key. We'd use this to create a host group per

From 4009a89239996ce6a447179695c0b9ae1b9ac170 Mon Sep 17 00:00:00 2001
From: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
Date: Tue, 23 Feb 2021 19:17:49 +0000
Subject: [PATCH 6/6] feat: add buf linting in CI

---
 .github/workflows/rust.yml | 12 +++++++++++-
 buf.yaml                   | 17 +++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 buf.yaml

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index a04455a1b9..b882ff9cc0 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -71,7 +71,7 @@ jobs:
           args: --workspace
 
   lints:
-    name: Lints
+    name: Rust Lints
     runs-on: ubuntu-latest
     container:
       image: quay.io/influxdb/rust:ci
@@ -91,3 +91,13 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --all-targets --workspace -- -D warnings
+
+  protobuf:
+    name: Protobuf Lints
+    runs-on: ubuntu-latest
+    container:
+      image: bufbuild/buf
+    steps:
+      - uses: actions/checkout@v2
+      - name: Lint IOx protobuf
+        run: buf lint
diff --git a/buf.yaml b/buf.yaml
new file mode 100644
index 0000000000..030255f9de
--- /dev/null
+++ b/buf.yaml
@@ -0,0 +1,17 @@
+version: v1beta1
+build:
+  roots:
+    - generated_types/protos/
+  excludes:
+    - generated_types/protos/com
+    - generated_types/protos/influxdata/platform
+
+lint:
+  use:
+    - DEFAULT
+    - STYLE_DEFAULT
+
+breaking:
+  use:
+    - WIRE
+    - WIRE_JSON