393 lines
10 KiB
Protocol Buffer
393 lines
10 KiB
Protocol Buffer
syntax = "proto3";
|
|
package influxdata.platform.storage;
|
|
|
|
import "google/protobuf/empty.proto";
|
|
import "google/protobuf/any.proto";
|
|
|
|
// TODO: how should requests handle authentication & authorization?
|
|
|
|
message CreateBucketRequest {
|
|
uint64 org_id = 1;
|
|
Bucket bucket = 2;
|
|
}
|
|
|
|
message CreateBucketResponse {
|
|
}
|
|
|
|
message DeleteBucketResponse {
|
|
}
|
|
|
|
message DeleteBucketRequest {
|
|
uint32 id = 1;
|
|
}
|
|
|
|
message GetBucketsResponse {
|
|
repeated Bucket buckets = 1;
|
|
}
|
|
|
|
message Organization {
|
|
uint64 id = 1;
|
|
string name = 2;
|
|
repeated Bucket buckets = 3;
|
|
}
|
|
|
|
message Bucket {
|
|
uint64 org_id = 1;
|
|
uint64 id = 2;
|
|
string name = 3;
|
|
string retention = 4;
|
|
// posting_list_rollover specifies how many series can be in a single posting list entry
|
|
// before the index creates a new entry for it. For example if you have _measurement=cpu and
|
|
// a rollover of 10, once 10 series are in that list, a new key/value pair is created in the
|
|
// index for the next list. So you'd have _measurement=cpu<big endian 0> and _measurement=cpu<big endian 10>
|
|
uint32 posting_list_rollover = 5;
|
|
repeated IndexLevel index_levels = 6;
|
|
}
|
|
|
|
message IndexLevel {
|
|
// duration is the amount of time this level represents. A duration of 0 is a level for an index of anything ever seen
|
|
uint32 duration_seconds = 1;
|
|
// timezone is the timezone under which this level should rollover. Only applicable for durations of 1 day or greater
|
|
string timezone = 2;
|
|
}
|
|
|
|
message S3PartitionRule {
|
|
|
|
}
|
|
|
|
service Delorean {
|
|
rpc CreateBucket(CreateBucketRequest) returns (CreateBucketResponse) {}
|
|
rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse) {}
|
|
rpc GetBuckets(Organization) returns (GetBucketsResponse) {}
|
|
}
|
|
|
|
// The following section is taken from InfluxDB so this server can implement the storage RPC. From here:
|
|
// https://github.com/influxdata/influxdb/blob/master/storage/reads/datatypes/predicate.proto
|
|
message Node {
|
|
enum Type {
|
|
LOGICAL_EXPRESSION = 0;
|
|
COMPARISON_EXPRESSION = 1;
|
|
PAREN_EXPRESSION = 2;
|
|
TAG_REF = 3;
|
|
LITERAL = 4;
|
|
FIELD_REF = 5;
|
|
}
|
|
|
|
enum Comparison {
|
|
EQUAL = 0;
|
|
NOT_EQUAL = 1;
|
|
STARTS_WITH = 2;
|
|
REGEX = 3;
|
|
NOT_REGEX = 4;
|
|
LT = 5;
|
|
LTE = 6;
|
|
GT = 7;
|
|
GTE = 8;
|
|
}
|
|
|
|
// Logical operators apply to boolean values and combine to produce a single boolean result.
|
|
enum Logical {
|
|
AND = 0;
|
|
OR = 1;
|
|
}
|
|
|
|
repeated Node children = 2;
|
|
|
|
oneof value {
|
|
string string_value = 3;
|
|
bool bool_value = 4;
|
|
int64 int_value = 5;
|
|
uint64 uint_value = 6;
|
|
double float_value = 7;
|
|
string regex_value = 8;
|
|
string tag_ref_value = 9;
|
|
string field_ref_value = 10;
|
|
Logical logical = 11;
|
|
Comparison comparison = 12;
|
|
}
|
|
}
|
|
|
|
message Predicate {
|
|
Node root = 1;
|
|
}
|
|
|
|
/*
|
|
S3 Organization scheme
|
|
|
|
Considerations:
|
|
* Buckets are tied to a region, so storage servers should use buckets from a specific region
|
|
|
|
one bucket per region. don't do per org as there is a default 100 bucket limit per account
|
|
|
|
prefix:
|
|
indexes have their own spot. They can be built based on snapshot levels, which are like granularities. So
|
|
<server_id>/<org_id>/<bucket_id>/index/<snapshot level>/
|
|
<server_id>/<org_id>/<bucket_id>/data/<start time><end time><id start><id end>.tsm
|
|
|
|
Some things we'd want to query:
|
|
measurement = cpu and host = serverA
|
|
tag keys where host = server a
|
|
tag keys where (app = foo and app = bar)
|
|
hosts where measurement = redis
|
|
fields where measurement = redis
|
|
hosts where (app = foo AND app = bar)
|
|
hosts
|
|
|
|
group data by measurement_range, time, size limit. Roll over to next measurement range when we hit either time or size
|
|
|
|
so we can say we want to snapshot to S3 at least every 1h or 100MB. Should also be able to force S3 snapshot.
|
|
|
|
We can have overlapping S3 data, which is fine. Need ability to request that the server looks at overlaps and compacts them
|
|
|
|
given a set of key/value pairs, determine which ranges we need to lookup.
|
|
|
|
are the set of ranges fixed or dynamic? If dynamic they have to be able to overlap. Which means we'll need to be able to compact them?
|
|
|
|
be able to tell the server how many requests to send to s3 in parallel to load
|
|
|
|
configure when to roll to deeper prefix (size/file count based)
|
|
|
|
on boot it should load up its S3 file index and keep in memory
|
|
*/
|
|
|
|
// From https://github.com/influxdata/idpe/blob/master/storage/read/service.proto
|
|
|
|
service Storage {
|
|
// ReadFilter performs a filter operation at storage
|
|
rpc ReadFilter (ReadFilterRequest) returns (stream ReadResponse);
|
|
|
|
// ReadGroup performs a group operation at storage
|
|
rpc ReadGroup (ReadGroupRequest) returns (stream ReadResponse);
|
|
|
|
// TagKeys performs a read operation for tag keys
|
|
rpc TagKeys (TagKeysRequest) returns (stream StringValuesResponse);
|
|
|
|
// TagValues performs a read operation for tag values
|
|
rpc TagValues (TagValuesRequest) returns (stream StringValuesResponse);
|
|
|
|
// Capabilities returns a map of keys and values identifying the capabilities supported by the storage engine
|
|
rpc Capabilities (google.protobuf.Empty) returns (CapabilitiesResponse);
|
|
|
|
rpc MeasurementNames(MeasurementNamesRequest) returns (stream StringValuesResponse);
|
|
|
|
rpc MeasurementTagKeys(MeasurementTagKeysRequest) returns (stream StringValuesResponse);
|
|
|
|
rpc MeasurementTagValues(MeasurementTagValuesRequest) returns (stream StringValuesResponse);
|
|
|
|
rpc MeasurementFields(MeasurementFieldsRequest) returns (stream MeasurementFieldsResponse);
|
|
}
|
|
|
|
// From https://github.com/influxdata/influxdb/blob/master/storage/reads/datatypes/storage_common.proto
|
|
|
|
message ReadFilterRequest {
|
|
google.protobuf.Any read_source = 1;
|
|
TimestampRange range = 2;
|
|
Predicate predicate = 3;
|
|
}
|
|
|
|
message ReadGroupRequest {
|
|
google.protobuf.Any read_source = 1;
|
|
TimestampRange range = 2;
|
|
Predicate predicate = 3;
|
|
|
|
enum Group {
|
|
// GroupNone returns all series as a single group.
|
|
// The single GroupFrame.TagKeys will be the union of all tag keys.
|
|
GROUP_NONE = 0;
|
|
|
|
// GroupBy returns a group for each unique value of the specified GroupKeys.
|
|
GROUP_BY = 2;
|
|
}
|
|
|
|
// GroupKeys specifies a list of tag keys used to order the data.
|
|
// It is dependent on the Group property to determine its behavior.
|
|
repeated string group_keys = 4;
|
|
|
|
Group group = 5;
|
|
Aggregate aggregate = 6;
|
|
}
|
|
|
|
message Aggregate {
|
|
enum AggregateType {
|
|
NONE = 0;
|
|
SUM = 1;
|
|
COUNT = 2;
|
|
MIN = 3;
|
|
MAX = 4;
|
|
}
|
|
|
|
AggregateType type = 1;
|
|
|
|
// additional arguments?
|
|
}
|
|
|
|
message Tag {
|
|
bytes key = 1;
|
|
bytes value = 2;
|
|
}
|
|
|
|
// Response message for ReadFilter and ReadGroup
|
|
message ReadResponse {
|
|
enum FrameType {
|
|
SERIES = 0;
|
|
POINTS = 1;
|
|
}
|
|
|
|
enum DataType {
|
|
FLOAT = 0;
|
|
INTEGER = 1;
|
|
UNSIGNED = 2;
|
|
BOOLEAN = 3;
|
|
STRING = 4;
|
|
}
|
|
|
|
message Frame {
|
|
oneof data {
|
|
GroupFrame group = 7;
|
|
SeriesFrame series = 1;
|
|
FloatPointsFrame float_points = 2;
|
|
IntegerPointsFrame integer_points = 3;
|
|
UnsignedPointsFrame unsigned_points = 4;
|
|
BooleanPointsFrame boolean_points = 5;
|
|
StringPointsFrame string_points = 6;
|
|
}
|
|
}
|
|
|
|
message GroupFrame {
|
|
// TagKeys
|
|
repeated bytes tag_keys = 1;
|
|
// PartitionKeyVals is the values of the partition key for this group, order matching ReadGroupRequest.GroupKeys
|
|
repeated bytes partition_key_vals = 2;
|
|
}
|
|
|
|
message SeriesFrame {
|
|
repeated Tag tags = 1;
|
|
DataType data_type = 2;
|
|
}
|
|
|
|
message FloatPointsFrame {
|
|
repeated sfixed64 timestamps = 1;
|
|
repeated double values = 2;
|
|
}
|
|
|
|
message IntegerPointsFrame {
|
|
repeated sfixed64 timestamps = 1;
|
|
repeated int64 values = 2;
|
|
}
|
|
|
|
message UnsignedPointsFrame {
|
|
repeated sfixed64 timestamps = 1;
|
|
repeated uint64 values = 2;
|
|
}
|
|
|
|
message BooleanPointsFrame {
|
|
repeated sfixed64 timestamps = 1;
|
|
repeated bool values = 2;
|
|
}
|
|
|
|
message StringPointsFrame {
|
|
repeated sfixed64 timestamps = 1;
|
|
repeated string values = 2;
|
|
}
|
|
|
|
repeated Frame frames = 1;
|
|
}
|
|
|
|
message CapabilitiesResponse {
|
|
map<string, string> caps = 1;
|
|
}
|
|
|
|
// Specifies a continuous range of nanosecond timestamps.
|
|
message TimestampRange {
|
|
// Start defines the inclusive lower bound.
|
|
int64 start = 1;
|
|
|
|
// End defines the exclusive upper bound.
|
|
int64 end = 2;
|
|
}
|
|
|
|
// TagKeysRequest is the request message for Storage.TagKeys.
|
|
message TagKeysRequest {
|
|
google.protobuf.Any tags_source = 1;
|
|
TimestampRange range = 2;
|
|
Predicate predicate = 3;
|
|
}
|
|
|
|
// TagValuesRequest is the request message for Storage.TagValues.
|
|
message TagValuesRequest {
|
|
google.protobuf.Any tags_source = 1;
|
|
TimestampRange range = 2;
|
|
Predicate predicate = 3;
|
|
string tag_key = 4;
|
|
}
|
|
|
|
// Response message for Storage.TagKeys, Storage.TagValues Storage.MeasurementNames,
|
|
// Storage.MeasurementTagKeys and Storage.MeasurementTagValues.
|
|
message StringValuesResponse {
|
|
repeated bytes values = 1;
|
|
}
|
|
|
|
// MeasurementNamesRequest is the request message for Storage.MeasurementNames.
|
|
message MeasurementNamesRequest {
|
|
google.protobuf.Any source = 1;
|
|
TimestampRange range = 2;
|
|
}
|
|
|
|
// MeasurementTagKeysRequest is the request message for Storage.MeasurementTagKeys.
|
|
message MeasurementTagKeysRequest {
|
|
google.protobuf.Any source = 1;
|
|
string measurement = 2;
|
|
TimestampRange range = 3;
|
|
Predicate predicate = 4;
|
|
}
|
|
|
|
// MeasurementTagValuesRequest is the request message for Storage.MeasurementTagValues.
|
|
message MeasurementTagValuesRequest {
|
|
google.protobuf.Any source = 1;
|
|
string measurement = 2;
|
|
string tag_key = 3;
|
|
TimestampRange range = 4;
|
|
Predicate predicate = 5;
|
|
}
|
|
|
|
// MeasurementFieldsRequest is the request message for Storage.MeasurementFields.
|
|
message MeasurementFieldsRequest {
|
|
google.protobuf.Any source = 1;
|
|
string measurement = 2;
|
|
TimestampRange range = 3;
|
|
Predicate predicate = 4;
|
|
}
|
|
|
|
// MeasurementFieldsResponse is the response message for Storage.MeasurementFields.
|
|
message MeasurementFieldsResponse {
|
|
enum FieldType {
|
|
FLOAT = 0;
|
|
INTEGER = 1;
|
|
UNSIGNED = 2;
|
|
STRING = 3;
|
|
BOOLEAN = 4;
|
|
UNDEFINED = 5;
|
|
}
|
|
|
|
message MessageField {
|
|
string key = 1;
|
|
FieldType type = 2;
|
|
sfixed64 timestamp = 3;
|
|
}
|
|
|
|
repeated MessageField fields = 1;
|
|
}
|
|
|
|
// From https://github.com/influxdata/idpe/blob/master/storage/read/source.proto
|
|
|
|
message ReadSource {
|
|
// OrgID specifies the organization identifier for this request.
|
|
uint64 org_id = 1;
|
|
|
|
// BucketID specifies the bucket in the organization.
|
|
uint64 bucket_id = 2;
|
|
|
|
// PartitionID specifies the partition to be queried.
|
|
uint64 partition_id = 3;
|
|
}
|