From 9bdc276d5ea8d5120a08278be0803bdd717bb2ac Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 8 Mar 2021 11:56:53 +0100 Subject: [PATCH 001/104] docs: Document how to use grpcurl to play with API --- README.md | 23 +++++++++++++++++++++++ scripts/grpcurl | 22 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 scripts/grpcurl diff --git a/README.md b/README.md index 3030b7d14c..9e7a0be302 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,29 @@ $ grpc_health_probe -addr 127.0.0.1:8082 -service influxdata.platform.storage.St status: SERVING ``` +### Manually calling gRPC API + +If you want to manually invoke one of the gRPC APIs, you can use any gRPC CLI client; +a good one is [grpcurl](https://github.com/fullstorydev/grpcurl). + +Tonic (the gRPC server library we're using) currently doesn't have support for gRPC reflection, +hence you must pass all `.proto` files to your client. You can find a conventient `grpcurl` wrapper +that does that in the `scripts` directory: + +```shell +$ ./scripts/grpcurl -plaintext 127.0.0.1:8082 list +grpc.health.v1.Health +influxdata.iox.management.v1.ManagementService +influxdata.platform.storage.IOxTesting +influxdata.platform.storage.Storage +$ ./scripts/grpcurl -plaintext 127.0.0.1:8082 influxdata.iox.management.v1.ManagementService.ListDatabases +{ + "names": [ + "foobar_weather" + ] +} +``` + ## Contributing We welcome community contributions from anyone! diff --git a/scripts/grpcurl b/scripts/grpcurl new file mode 100755 index 0000000000..36fa4a6ce6 --- /dev/null +++ b/scripts/grpcurl @@ -0,0 +1,22 @@ +#!/bin/bash +# +# This script is a convenience wrapper around grpcurl that passes all the known *.proto to it. +# +# Script self-destruction condition: +# Once tonic implements reflection this script will no longer be necessary. +# The reflection feature is tracked in https://github.com/hyperium/tonic/issues/165 +# and marked as closed and will likely be included in a tonic version > 0.4.0. +# + +set -eu -o pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +proto_dir="${SCRIPT_DIR}"/../generated_types/protos + +# bash 3.x (default on macos big-sur 🤦) has no readarray. +while IFS= read -r line; do + proto_flags+=("-proto" "$line") +done < <(find "${proto_dir}" -name '*.proto') + +grpcurl -import-path ./generated_types/protos "${proto_flags[@]}" "$@" From f2642d6fbc13b3a2602076b7f1d6840b0aa60c52 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 8 Mar 2021 15:52:18 +0100 Subject: [PATCH 002/104] docs: Use console type for interactive shell snippets --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9e7a0be302..0dfacaa68b 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Should you desire specifying config via a file, you can do so using a `.env` formatted file in the working directory. You can use the provided [example](docs/env.example) as a template if you want: -```bash +```shell cp docs/env.example .env ``` @@ -194,7 +194,7 @@ This can be set three ways: - set an environment variable `INFLUXDB_IOX_ID=42` - set a flag `--writer-id 42` - send an HTTP PUT request: -``` +```shell curl --request PUT \ --url http://localhost:8080/iox/api/v1/id \ --header 'Content-Type: application/json' \ @@ -205,7 +205,7 @@ curl --request PUT \ To write data, you need a destination database. This is set via HTTP PUT, identifying the database by org `company` and bucket `sensors`: -``` +```shell curl --request PUT \ --url http://localhost:8080/iox/api/v1/databases/company_sensors \ --header 'Content-Type: application/json' \ @@ -237,14 +237,14 @@ curl -v -G -d 'org=company' -d 'bucket=sensors' --data-urlencode 'sql_query=sele The HTTP API exposes a healthcheck endpoint at `/health` -```shell +```console $ curl http://127.0.0.1:8080/health OK ``` The gRPC API implements the [gRPC Health Checking Protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). This can be tested with [grpc-health-probe](https://github.com/grpc-ecosystem/grpc-health-probe) -```shell +```console $ grpc_health_probe -addr 127.0.0.1:8082 -service influxdata.platform.storage.Storage status: SERVING ``` @@ -258,7 +258,7 @@ Tonic (the gRPC server library we're using) currently doesn't have support for g hence you must pass all `.proto` files to your client. You can find a conventient `grpcurl` wrapper that does that in the `scripts` directory: -```shell +```console $ ./scripts/grpcurl -plaintext 127.0.0.1:8082 list grpc.health.v1.Health influxdata.iox.management.v1.ManagementService From 500c237f62bd27cc4f4249301e108075ebbbab4c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 8 Mar 2021 12:35:19 -0500 Subject: [PATCH 003/104] feat: Add Service + CLI command to write line protocol to IOx (#939) * feat: Add Service + CLI command to write line protocol to IOx * fix: clippy * fix: protobuf lint * fix: LinePrtocol typo --- generated_types/build.rs | 2 + .../influxdata/iox/write/v1/service.proto | 23 ++++++ generated_types/src/lib.rs | 9 +++ google_types/src/lib.rs | 6 ++ influxdb_iox_client/src/client.rs | 3 + influxdb_iox_client/src/client/write.rs | 77 +++++++++++++++++++ influxdb_iox_client/src/lib.rs | 2 +- src/commands/database.rs | 51 +++++++++++- src/influxdb_ioxd/rpc.rs | 3 + src/influxdb_ioxd/rpc/error.rs | 26 +++++++ src/influxdb_ioxd/rpc/management.rs | 24 +----- src/influxdb_ioxd/rpc/write.rs | 60 +++++++++++++++ tests/end-to-end.rs | 4 +- tests/end_to_end_cases/management_api.rs | 12 +-- tests/end_to_end_cases/mod.rs | 3 + tests/end_to_end_cases/util.rs | 10 +++ tests/end_to_end_cases/write_api.rs | 71 +++++++++++++++++ tests/end_to_end_cases/write_cli.rs | 63 +++++++++++++++ 18 files changed, 414 insertions(+), 35 deletions(-) create mode 100644 generated_types/protos/influxdata/iox/write/v1/service.proto create mode 100644 influxdb_iox_client/src/client/write.rs create mode 100644 src/influxdb_ioxd/rpc/error.rs create mode 100644 src/influxdb_ioxd/rpc/write.rs create mode 100644 tests/end_to_end_cases/util.rs create mode 100644 tests/end_to_end_cases/write_api.rs create mode 100644 tests/end_to_end_cases/write_cli.rs diff --git a/generated_types/build.rs b/generated_types/build.rs index 57261e1558..c864c96eeb 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -28,6 +28,7 @@ fn generate_grpc_types(root: &Path) -> Result<()> { let storage_path = root.join("influxdata/platform/storage"); let idpe_path = root.join("com/github/influxdata/idpe/storage/read"); let management_path = root.join("influxdata/iox/management/v1"); + let write_path = root.join("influxdata/iox/write/v1"); let grpc_path = root.join("grpc/health/v1"); let proto_files = vec![ @@ -40,6 +41,7 @@ fn generate_grpc_types(root: &Path) -> Result<()> { management_path.join("base_types.proto"), management_path.join("database_rules.proto"), management_path.join("service.proto"), + write_path.join("service.proto"), grpc_path.join("service.proto"), ]; diff --git a/generated_types/protos/influxdata/iox/write/v1/service.proto b/generated_types/protos/influxdata/iox/write/v1/service.proto new file mode 100644 index 0000000000..9476204780 --- /dev/null +++ b/generated_types/protos/influxdata/iox/write/v1/service.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; +package influxdata.iox.write.v1; + + +service WriteService { + // write data into a specific Database + rpc Write(WriteRequest) returns (WriteResponse); +} + +message WriteRequest { + // name of database into which to write + string name = 1; + + // data, in [LineProtocol] format + // + // [LineProtocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format) + string lp_data = 2; +} + +message WriteResponse { + // how many lines were parsed and written into the database + uint64 lines_written = 1; +} diff --git a/generated_types/src/lib.rs b/generated_types/src/lib.rs index 0cb21817ce..ce599dd48d 100644 --- a/generated_types/src/lib.rs +++ b/generated_types/src/lib.rs @@ -9,6 +9,9 @@ clippy::clone_on_ref_ptr )] +/// This module imports the generated protobuf code into a Rust module +/// heirarchy that matches the namespace heirarchy of the protobuf +/// definitions mod pb { pub mod influxdata { pub mod platform { @@ -33,6 +36,12 @@ mod pb { include!(concat!(env!("OUT_DIR"), "/influxdata.iox.management.v1.rs")); } } + + pub mod write { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/influxdata.iox.write.v1.rs")); + } + } } } diff --git a/google_types/src/lib.rs b/google_types/src/lib.rs index 74ad9b76e2..e9c6ab1458 100644 --- a/google_types/src/lib.rs +++ b/google_types/src/lib.rs @@ -1,3 +1,6 @@ +//! Protobuf types for errors from the google standards and +//! conversions to `tonic::Status` + // This crate deliberately does not use the same linting rules as the other // crates because of all the generated code it contains that we don't have much // control over. @@ -87,6 +90,9 @@ fn encode_status(code: tonic::Code, message: String, details: Any) -> tonic::Sta } #[derive(Debug, Default, Clone)] +/// Error returned if a request field has an invalid value. Includes +/// machinery to add parent field names for context -- thus it will +/// report `rules.write_timeout` than simply `write_timeout`. pub struct FieldViolation { pub field: String, pub description: String, diff --git a/influxdb_iox_client/src/client.rs b/influxdb_iox_client/src/client.rs index 01addd884e..53ba6ead6c 100644 --- a/influxdb_iox_client/src/client.rs +++ b/influxdb_iox_client/src/client.rs @@ -4,6 +4,9 @@ pub mod health; /// Client for the management API pub mod management; +/// Client for the write API +pub mod write; + #[cfg(feature = "flight")] /// Client for the flight API pub mod flight; diff --git a/influxdb_iox_client/src/client/write.rs b/influxdb_iox_client/src/client/write.rs new file mode 100644 index 0000000000..cea5ad12a8 --- /dev/null +++ b/influxdb_iox_client/src/client/write.rs @@ -0,0 +1,77 @@ +use thiserror::Error; + +use self::generated_types::{write_service_client::WriteServiceClient, *}; + +use crate::connection::Connection; + +/// Re-export generated_types +pub mod generated_types { + pub use generated_types::influxdata::iox::write::v1::*; +} + +/// Errors returned by Client::write_data +#[derive(Debug, Error)] +pub enum WriteError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// An IOx Write API client. +/// +/// ```no_run +/// #[tokio::main] +/// # async fn main() { +/// use influxdb_iox_client::{ +/// write::Client, +/// connection::Builder, +/// }; +/// +/// let mut connection = Builder::default() +/// .build("http://127.0.0.1:8082") +/// .await +/// .unwrap(); +/// +/// let mut client = Client::new(connection); +/// +/// // write a line of line procol data +/// client +/// .write("bananas", "cpu,region=west user=23.2 100") +/// .await +/// .expect("failed to create database"); +/// # } +/// ``` +#[derive(Debug, Clone)] +pub struct Client { + inner: WriteServiceClient, +} + +impl Client { + /// Creates a new client with the provided connection + pub fn new(channel: tonic::transport::Channel) -> Self { + Self { + inner: WriteServiceClient::new(channel), + } + } + + /// Write the [LineProtocol] formatted data in `lp_data` to + /// database `name`. Returns the number of lines which were parsed + /// and written to the database + /// + /// [LineProtocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format) + pub async fn write( + &mut self, + name: impl Into, + lp_data: impl Into, + ) -> Result { + let name = name.into(); + let lp_data = lp_data.into(); + let response = self + .inner + .write(WriteRequest { name, lp_data }) + .await + .map_err(WriteError::ServerError)?; + + Ok(response.into_inner().lines_written as usize) + } +} diff --git a/influxdb_iox_client/src/lib.rs b/influxdb_iox_client/src/lib.rs index 5686cfab00..ddb713535b 100644 --- a/influxdb_iox_client/src/lib.rs +++ b/influxdb_iox_client/src/lib.rs @@ -8,7 +8,7 @@ )] #![allow(clippy::missing_docs_in_private_items)] -pub use client::{health, management}; +pub use client::{health, management, write}; #[cfg(feature = "flight")] pub use client::flight; diff --git a/src/commands/database.rs b/src/commands/database.rs index d128f03a69..3ec3a6e149 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -1,6 +1,12 @@ +//! This module implements the `database` CLI command +use std::{fs::File, io::Read, path::PathBuf}; + use influxdb_iox_client::{ connection::Builder, - management::{generated_types::*, *}, + management::{ + self, generated_types::*, CreateDatabaseError, GetDatabaseError, ListDatabaseError, + }, + write::{self, WriteError}, }; use structopt::StructOpt; use thiserror::Error; @@ -18,6 +24,15 @@ pub enum Error { #[error("Error connecting to IOx: {0}")] ConnectionError(#[from] influxdb_iox_client::connection::Error), + + #[error("Error reading file {:?}: {}", file_name, source)] + ReadingFile { + file_name: PathBuf, + source: std::io::Error, + }, + + #[error("Error writing: {0}")] + WriteError(#[from] WriteError), } pub type Result = std::result::Result; @@ -47,18 +62,30 @@ struct Get { name: Option, } +/// Write data into the specified database +#[derive(Debug, StructOpt)] +struct Write { + /// The name of the database + name: String, + + /// File with data to load. Currently supported formats are .lp + file_name: PathBuf, +} + +/// All possible subcommands for database #[derive(Debug, StructOpt)] enum Command { Create(Create), Get(Get), + Write(Write), } pub async fn command(url: String, config: Config) -> Result<()> { let connection = Builder::default().build(url).await?; - let mut client = Client::new(connection); match config.command { Command::Create(command) => { + let mut client = management::Client::new(connection); client .create_database(DatabaseRules { name: command.name, @@ -74,6 +101,7 @@ pub async fn command(url: String, config: Config) -> Result<()> { println!("Ok"); } Command::Get(get) => { + let mut client = management::Client::new(connection); if let Some(name) = get.name { let database = client.get_database(name).await?; // TOOD: Do something better than this @@ -83,6 +111,25 @@ pub async fn command(url: String, config: Config) -> Result<()> { println!("{}", databases.join(", ")) } } + Command::Write(write) => { + let mut client = write::Client::new(connection); + + let mut file = File::open(&write.file_name).map_err(|e| Error::ReadingFile { + file_name: write.file_name.clone(), + source: e, + })?; + + let mut lp_data = String::new(); + file.read_to_string(&mut lp_data) + .map_err(|e| Error::ReadingFile { + file_name: write.file_name.clone(), + source: e, + })?; + + let lines_written = client.write(write.name, lp_data).await?; + + println!("{} Lines OK", lines_written); + } } Ok(()) diff --git a/src/influxdb_ioxd/rpc.rs b/src/influxdb_ioxd/rpc.rs index 28e2da4c7e..7998674235 100644 --- a/src/influxdb_ioxd/rpc.rs +++ b/src/influxdb_ioxd/rpc.rs @@ -8,10 +8,12 @@ use tokio_stream::wrappers::TcpListenerStream; use data_types::error::ErrorLogger; use server::{ConnectionManager, Server}; +pub mod error; mod flight; mod management; mod storage; mod testing; +mod write; #[derive(Debug, Snafu)] pub enum Error { @@ -50,6 +52,7 @@ where .add_service(testing::make_server()) .add_service(storage::make_server(Arc::clone(&server))) .add_service(flight::make_server(Arc::clone(&server))) + .add_service(write::make_server(Arc::clone(&server))) .add_service(management::make_server(server)) .serve_with_incoming(stream) .await diff --git a/src/influxdb_ioxd/rpc/error.rs b/src/influxdb_ioxd/rpc/error.rs new file mode 100644 index 0000000000..8c80714a01 --- /dev/null +++ b/src/influxdb_ioxd/rpc/error.rs @@ -0,0 +1,26 @@ +use generated_types::google::{InternalError, NotFound, PreconditionViolation}; +use tracing::error; + +use server::Error; + +/// map common server errors to the appropriate tonic Status +pub fn default_error_handler(error: Error) -> tonic::Status { + match error { + Error::IdNotSet => PreconditionViolation { + category: "Writer ID".to_string(), + subject: "influxdata.com/iox".to_string(), + description: "Writer ID must be set".to_string(), + } + .into(), + Error::DatabaseNotFound { db_name } => NotFound { + resource_type: "database".to_string(), + resource_name: db_name, + ..Default::default() + } + .into(), + error => { + error!(?error, "Unexpected error"); + InternalError {}.into() + } + } +} diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index b45ab28ba5..102b8b7c2b 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -2,37 +2,19 @@ use std::convert::TryInto; use std::fmt::Debug; use std::sync::Arc; -use tonic::{Request, Response, Status}; -use tracing::error; - use data_types::database_rules::DatabaseRules; use data_types::DatabaseName; -use generated_types::google::{ - AlreadyExists, FieldViolation, FieldViolationExt, InternalError, NotFound, - PreconditionViolation, -}; +use generated_types::google::{AlreadyExists, FieldViolation, FieldViolationExt, NotFound}; use generated_types::influxdata::iox::management::v1::*; use query::DatabaseStore; use server::{ConnectionManager, Error, Server}; +use tonic::{Request, Response, Status}; struct ManagementService { server: Arc>, } -fn default_error_handler(error: Error) -> tonic::Status { - match error { - Error::IdNotSet => PreconditionViolation { - category: "Writer ID".to_string(), - subject: "influxdata.com/iox".to_string(), - description: "Writer ID must be set".to_string(), - } - .into(), - error => { - error!(?error, "Unexpected error"); - InternalError {}.into() - } - } -} +use super::error::default_error_handler; #[tonic::async_trait] impl management_service_server::ManagementService for ManagementService diff --git a/src/influxdb_ioxd/rpc/write.rs b/src/influxdb_ioxd/rpc/write.rs new file mode 100644 index 0000000000..2b3eafaea1 --- /dev/null +++ b/src/influxdb_ioxd/rpc/write.rs @@ -0,0 +1,60 @@ +use std::sync::Arc; + +use generated_types::{google::FieldViolation, influxdata::iox::write::v1::*}; +use influxdb_line_protocol::parse_lines; +use server::{ConnectionManager, Server}; +use std::fmt::Debug; +use tonic::Response; +use tracing::debug; + +use super::error::default_error_handler; + +/// Implementation of the write service +struct WriteService { + server: Arc>, +} + +#[tonic::async_trait] +impl write_service_server::WriteService for WriteService +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + async fn write( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); + + let db_name = request.name; + let lp_data = request.lp_data; + let lp_chars = lp_data.len(); + + let lines = parse_lines(&lp_data) + .collect::, influxdb_line_protocol::Error>>() + .map_err(|e| FieldViolation { + field: "lp_data".into(), + description: format!("Invalid Line Protocol: {}", e), + })?; + + let lp_line_count = lines.len(); + debug!(%db_name, %lp_chars, lp_line_count, "Writing lines into database"); + + self.server + .write_lines(&db_name, &lines) + .await + .map_err(default_error_handler)?; + + let lines_written = lp_line_count as u64; + Ok(Response::new(WriteResponse { lines_written })) + } +} + +/// Instantiate the write service +pub fn make_server( + server: Arc>, +) -> write_service_server::WriteServiceServer +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + write_service_server::WriteServiceServer::new(WriteService { server }) +} diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index f31a3c521a..4125807097 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -76,7 +76,7 @@ async fn read_and_write_data() { .await .unwrap(); let mut storage_client = StorageClient::new(grpc.clone()); - let mut management_client = influxdb_iox_client::management::Client::new(grpc); + let mut management_client = influxdb_iox_client::management::Client::new(grpc.clone()); // These tests share data; TODO: a better way to indicate this { @@ -104,6 +104,8 @@ async fn read_and_write_data() { .await; management_api::test(&mut management_client).await; management_cli::test(GRPC_URL_BASE).await; + write_api::test(grpc).await; + write_cli::test(GRPC_URL_BASE).await; test_http_error_messages(&influxdb2).await.unwrap(); } diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 7992f25c64..0cc9145e62 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -1,11 +1,11 @@ use std::num::NonZeroU32; -use rand::{distributions::Alphanumeric, thread_rng, Rng}; - use generated_types::google::protobuf::Empty; use generated_types::{google::protobuf::Duration, influxdata::iox::management::v1::*}; use influxdb_iox_client::management::{Client, CreateDatabaseError}; +use super::util::rand_name; + pub async fn test(client: &mut Client) { test_set_get_writer_id(client).await; test_create_database_duplicate_name(client).await; @@ -149,11 +149,3 @@ async fn test_create_get_database(client: &mut Client) { assert_eq!(response, rules); } - -fn rand_name() -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .take(10) - .map(char::from) - .collect() -} diff --git a/tests/end_to_end_cases/mod.rs b/tests/end_to_end_cases/mod.rs index 99fb1b6459..8584f185f4 100644 --- a/tests/end_to_end_cases/mod.rs +++ b/tests/end_to_end_cases/mod.rs @@ -3,3 +3,6 @@ pub mod management_api; pub mod management_cli; pub mod read_api; pub mod storage_api; +pub mod util; +pub mod write_api; +pub mod write_cli; diff --git a/tests/end_to_end_cases/util.rs b/tests/end_to_end_cases/util.rs new file mode 100644 index 0000000000..f70c1cd241 --- /dev/null +++ b/tests/end_to_end_cases/util.rs @@ -0,0 +1,10 @@ +use rand::{distributions::Alphanumeric, thread_rng, Rng}; + +/// Return a random string suitable for use as a database name +pub fn rand_name() -> String { + thread_rng() + .sample_iter(&Alphanumeric) + .take(10) + .map(char::from) + .collect() +} diff --git a/tests/end_to_end_cases/write_api.rs b/tests/end_to_end_cases/write_api.rs new file mode 100644 index 0000000000..49250767cb --- /dev/null +++ b/tests/end_to_end_cases/write_api.rs @@ -0,0 +1,71 @@ +use std::num::NonZeroU32; + +use influxdb_iox_client::management::{self, generated_types::DatabaseRules}; +use influxdb_iox_client::write::{self, WriteError}; +use test_helpers::assert_contains; + +use super::util::rand_name; + +/// Tests the basics of the write API +pub async fn test(channel: tonic::transport::Channel) { + let mut management_client = management::Client::new(channel.clone()); + let mut write_client = write::Client::new(channel); + + test_write(&mut management_client, &mut write_client).await +} + +async fn test_write(management_client: &mut management::Client, write_client: &mut write::Client) { + const TEST_ID: u32 = 42; + + let db_name = rand_name(); + + management_client + .create_database(DatabaseRules { + name: db_name.clone(), + ..Default::default() + }) + .await + .expect("create database failed"); + + management_client + .update_writer_id(NonZeroU32::new(TEST_ID).unwrap()) + .await + .expect("set ID failed"); + + let lp_lines = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + "disk,region=east bytes=99i 200", + ]; + + let num_lines_written = write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + assert_eq!(num_lines_written, 3); + + // ---- test bad data ---- + let err = write_client + .write(&db_name, "XXX") + .await + .expect_err("expected write to fail"); + + assert_contains!( + err.to_string(), + r#"Client specified an invalid argument: Violation for field "lp_data": Invalid Line Protocol: A generic parsing error occurred"# + ); + assert!(matches!(dbg!(err), WriteError::ServerError(_))); + + // ---- test non existent database ---- + let err = write_client + .write("Non_existent_database", lp_lines.join("\n")) + .await + .expect_err("expected write to fail"); + + assert_contains!( + err.to_string(), + r#"Unexpected server error: Some requested entity was not found: Resource database/Non_existent_database not found"# + ); + assert!(matches!(dbg!(err), WriteError::ServerError(_))); +} diff --git a/tests/end_to_end_cases/write_cli.rs b/tests/end_to_end_cases/write_cli.rs new file mode 100644 index 0000000000..575f97da4a --- /dev/null +++ b/tests/end_to_end_cases/write_cli.rs @@ -0,0 +1,63 @@ +use assert_cmd::Command; +use predicates::prelude::*; +use test_helpers::make_temp_file; + +use super::util::rand_name; + +pub async fn test(addr: impl AsRef) { + let db_name = rand_name(); + let addr = addr.as_ref(); + create_database(&db_name, addr).await; + test_write(&db_name, addr).await; +} + +async fn create_database(db_name: &str, addr: &str) { + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); +} + +async fn test_write(db_name: &str, addr: &str) { + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + ]; + + let lp_data_file = make_temp_file(lp_data.join("\n")); + + // read from temp file + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(db_name) + .arg(lp_data_file.as_ref()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("2 Lines OK")); + + // try reading a non existent file + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(db_name) + .arg("this_file_does_not_exist") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr( + predicate::str::contains(r#"Error reading file "this_file_does_not_exist":"#) + .and(predicate::str::contains("No such file or directory")), + ); +} From ac1b0c04ae4876513b0be198a08c10a92cdee20d Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Tue, 2 Mar 2021 14:08:22 -0800 Subject: [PATCH 004/104] fix(line-protocol): add unsigned integer field type Fixes #904 The line protocol parser was lacking the unsigned integer type, which suffixes values with `u`. This adds unsigned integer support to the line protocol parser, and fills a few corresponding gaps in the mutable buffer. --- data_types/src/data.rs | 11 +++++++++ influxdb_line_protocol/src/lib.rs | 37 ++++++++++++++++++++++--------- ingest/src/lib.rs | 4 ++++ mutable_buffer/src/column.rs | 30 +++++++++++++++++++++++++ mutable_buffer/src/table.rs | 14 +++++++++++- packers/src/packers.rs | 10 +++++++++ 6 files changed, 95 insertions(+), 11 deletions(-) diff --git a/data_types/src/data.rs b/data_types/src/data.rs index f511c5f47f..20bd1c66b1 100644 --- a/data_types/src/data.rs +++ b/data_types/src/data.rs @@ -317,6 +317,7 @@ fn add_line<'a>( for (column, value) in &line.field_set { let val = match value { FieldValue::I64(v) => add_i64_value(fbb, column.as_str(), *v), + FieldValue::U64(v) => add_u64_value(fbb, column.as_str(), *v), FieldValue::F64(v) => add_f64_value(fbb, column.as_str(), *v), FieldValue::Boolean(v) => add_bool_value(fbb, column.as_str(), *v), FieldValue::String(v) => add_string_value(fbb, column.as_str(), v.as_str()), @@ -393,6 +394,16 @@ fn add_i64_value<'a>( add_value(fbb, column, wb::ColumnValue::I64Value, iv.as_union_value()) } +fn add_u64_value<'a>( + fbb: &mut FlatBufferBuilder<'a>, + column: &str, + value: u64, +) -> flatbuffers::WIPOffset> { + let iv = wb::U64Value::create(fbb, &wb::U64ValueArgs { value }); + + add_value(fbb, column, wb::ColumnValue::U64Value, iv.as_union_value()) +} + fn add_bool_value<'a>( fbb: &mut FlatBufferBuilder<'a>, column: &str, diff --git a/influxdb_line_protocol/src/lib.rs b/influxdb_line_protocol/src/lib.rs index 6b0629bb46..f38d2d8f73 100644 --- a/influxdb_line_protocol/src/lib.rs +++ b/influxdb_line_protocol/src/lib.rs @@ -50,6 +50,12 @@ pub enum Error { value: String, }, + #[snafu(display(r#"Unable to parse unsigned integer value '{}'"#, value))] + UIntegerValueInvalid { + source: std::num::ParseIntError, + value: String, + }, + #[snafu(display(r#"Unable to parse floating-point value '{}'"#, value))] FloatValueInvalid { source: std::num::ParseFloatError, @@ -333,10 +339,11 @@ pub type FieldSet<'a> = SmallVec<[(EscapedStr<'a>, FieldValue<'a>); 4]>; pub type TagSet<'a> = SmallVec<[(EscapedStr<'a>, EscapedStr<'a>); 8]>; /// Allowed types of Fields in a `ParsedLine`. One of the types described in -/// https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/#data-types +/// https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format #[derive(Debug, Clone, PartialEq)] pub enum FieldValue<'a> { I64(i64), + U64(u64), F64(f64), String(EscapedStr<'a>), Boolean(bool), @@ -349,6 +356,7 @@ impl<'a> Display for FieldValue<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::I64(v) => write!(f, "{}i", v), + Self::U64(v) => write!(f, "{}u", v), Self::F64(v) => write!(f, "{}", v), Self::String(v) => escape_and_write_value(f, v, FIELD_VALUE_STRING_DELIMITERS), Self::Boolean(v) => write!(f, "{}", v), @@ -644,20 +652,28 @@ fn field_key(i: &str) -> IResult<&str, EscapedStr<'_>> { fn field_value(i: &str) -> IResult<&str, FieldValue<'_>> { let int = map(field_integer_value, FieldValue::I64); + let uint = map(field_uinteger_value, FieldValue::U64); let float = map(field_float_value, FieldValue::F64); let string = map(field_string_value, FieldValue::String); let boolv = map(field_bool_value, FieldValue::Boolean); - alt((int, float, string, boolv))(i) + alt((int, uint, float, string, boolv))(i) } fn field_integer_value(i: &str) -> IResult<&str, i64> { - let tagged_value = terminated(integral_value_common, tag("i")); + let tagged_value = terminated(integral_value_signed, tag("i")); map_fail(tagged_value, |value| { value.parse().context(IntegerValueInvalid { value }) })(i) } +fn field_uinteger_value(i: &str) -> IResult<&str, u64> { + let tagged_value = terminated(digit1, tag("u")); + map_fail(tagged_value, |value| { + value.parse().context(UIntegerValueInvalid { value }) + })(i) +} + fn field_float_value(i: &str) -> IResult<&str, f64> { let value = alt((field_float_value_with_decimal, field_float_value_no_decimal)); map_fail(value, |value| { @@ -666,25 +682,25 @@ fn field_float_value(i: &str) -> IResult<&str, f64> { } fn field_float_value_with_decimal(i: &str) -> IResult<&str, &str> { - recognize(separated_pair(integral_value_common, tag("."), digit1))(i) + recognize(separated_pair(integral_value_signed, tag("."), digit1))(i) } fn field_float_value_no_decimal(i: &str) -> IResult<&str, &str> { - integral_value_common(i) + integral_value_signed(i) } -fn integral_value_common(i: &str) -> IResult<&str, &str> { +fn integral_value_signed(i: &str) -> IResult<&str, &str> { recognize(preceded(opt(tag("-")), digit1))(i) } fn timestamp(i: &str) -> IResult<&str, i64> { - map_fail(integral_value_common, |value| { + map_fail(integral_value_signed, |value| { value.parse().context(TimestampValueInvalid { value }) })(i) } fn field_string_value(i: &str) -> IResult<&str, EscapedStr<'_>> { - // https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/#data-types + // https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format // For string field values, backslash is only used to escape itself(\) or double // quotes. let string_data = alt(( @@ -707,7 +723,7 @@ fn field_string_value(i: &str) -> IResult<&str, EscapedStr<'_>> { } fn field_bool_value(i: &str) -> IResult<&str, bool> { - // https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/#data-types + // https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format // "specify TRUE with t, T, true, True, or TRUE. Specify FALSE with f, F, false, // False, or FALSE alt(( @@ -1913,7 +1929,8 @@ her"#, #[test] fn field_value_display() -> Result { - assert_eq!(FieldValue::I64(42).to_string(), "42i"); + assert_eq!(FieldValue::I64(-42).to_string(), "-42i"); + assert_eq!(FieldValue::U64(42).to_string(), "42u"); assert_eq!(FieldValue::F64(42.11).to_string(), "42.11"); assert_eq!( FieldValue::String(EscapedStr::from("foo")).to_string(), diff --git a/ingest/src/lib.rs b/ingest/src/lib.rs index 37f45a1c82..637bdf7c58 100644 --- a/ingest/src/lib.rs +++ b/ingest/src/lib.rs @@ -310,6 +310,7 @@ impl<'a> MeasurementSampler<'a> { let field_type = match field_value { FieldValue::F64(_) => InfluxFieldType::Float, FieldValue::I64(_) => InfluxFieldType::Integer, + FieldValue::U64(_) => InfluxFieldType::UInteger, FieldValue::String(_) => InfluxFieldType::String, FieldValue::Boolean(_) => InfluxFieldType::Boolean, }; @@ -474,6 +475,9 @@ fn pack_lines<'a>(schema: &Schema, lines: &[ParsedLine<'a>]) -> Vec { FieldValue::I64(i) => { packer.i64_packer_mut().push(i); } + FieldValue::U64(i) => { + packer.u64_packer_mut().push(i); + } FieldValue::String(ref s) => { packer.bytes_packer_mut().push(ByteArray::from(s.as_str())); } diff --git a/mutable_buffer/src/column.rs b/mutable_buffer/src/column.rs index ab6e2e9991..d10c6e2c78 100644 --- a/mutable_buffer/src/column.rs +++ b/mutable_buffer/src/column.rs @@ -34,6 +34,7 @@ pub type Result = std::result::Result; pub enum Column { F64(Vec>, StatValues), I64(Vec>, StatValues), + U64(Vec>, StatValues), String(Vec>, StatValues), Bool(Vec>, StatValues), Tag(Vec>, StatValues), @@ -66,6 +67,15 @@ impl Column { vals.push(Some(val)); Self::I64(vals, StatValues::new(val)) } + U64Value => { + let val = value + .value_as_u64value() + .expect("u64 value should be present") + .value(); + let mut vals = vec![None; capacity]; + vals.push(Some(val)); + Self::U64(vals, StatValues::new(val)) + } StringValue => { let val = value .value_as_string_value() @@ -109,6 +119,7 @@ impl Column { match self { Self::F64(v, _) => v.len(), Self::I64(v, _) => v.len(), + Self::U64(v, _) => v.len(), Self::String(v, _) => v.len(), Self::Bool(v, _) => v.len(), Self::Tag(v, _) => v.len(), @@ -123,6 +134,7 @@ impl Column { match self { Self::F64(_, _) => "f64", Self::I64(_, _) => "i64", + Self::U64(_, _) => "u64", Self::String(_, _) => "String", Self::Bool(_, _) => "bool", Self::Tag(_, _) => "tag", @@ -134,6 +146,7 @@ impl Column { match self { Self::F64(..) => ArrowDataType::Float64, Self::I64(..) => ArrowDataType::Int64, + Self::U64(..) => ArrowDataType::UInt64, Self::String(..) => ArrowDataType::Utf8, Self::Bool(..) => ArrowDataType::Boolean, Self::Tag(..) => ArrowDataType::Utf8, @@ -179,6 +192,15 @@ impl Column { } None => false, }, + Self::U64(vals, stats) => match value.value_as_u64value() { + Some(u64_val) => { + let u64_val = u64_val.value(); + vals.push(Some(u64_val)); + stats.update(u64_val); + true + } + None => false, + }, Self::F64(vals, stats) => match value.value_as_f64value() { Some(f64_val) => { let f64_val = f64_val.value(); @@ -216,6 +238,11 @@ impl Column { v.push(None); } } + Self::U64(v, _) => { + if v.len() == len { + v.push(None); + } + } Self::String(v, _) => { if v.len() == len { v.push(None); @@ -290,6 +317,9 @@ impl Column { Self::I64(v, stats) => { mem::size_of::>() * v.len() + mem::size_of_val(&stats) } + Self::U64(v, stats) => { + mem::size_of::>() * v.len() + mem::size_of_val(&stats) + } Self::Bool(v, stats) => { mem::size_of::>() * v.len() + mem::size_of_val(&stats) } diff --git a/mutable_buffer/src/table.rs b/mutable_buffer/src/table.rs index 9174a1c99d..35e1458dd5 100644 --- a/mutable_buffer/src/table.rs +++ b/mutable_buffer/src/table.rs @@ -23,7 +23,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use arrow_deps::{ arrow, arrow::{ - array::{ArrayRef, BooleanBuilder, Float64Builder, Int64Builder, StringBuilder}, + array::{ArrayRef, BooleanBuilder, Float64Builder, Int64Builder, UInt64Builder, StringBuilder}, datatypes::DataType as ArrowDataType, record_batch::RecordBatch, }, @@ -326,6 +326,7 @@ impl Table { schema_builder.field(column_name, ArrowDataType::Int64) } } + Column::U64(_, _) => schema_builder.field(column_name, ArrowDataType::UInt64), Column::Bool(_, _) => schema_builder.field(column_name, ArrowDataType::Boolean), }; } @@ -399,6 +400,15 @@ impl Table { Arc::new(builder.finish()) as ArrayRef } + Column::U64(vals, _) => { + let mut builder = UInt64Builder::new(vals.len()); + + for v in vals { + builder.append_option(*v).context(ArrowError {})?; + } + + Arc::new(builder.finish()) as ArrayRef + } Column::Bool(vals, _) => { let mut builder = BooleanBuilder::new(vals.len()); @@ -504,6 +514,7 @@ impl Table { match column { Column::F64(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::I64(v, _) => self.column_value_matches_predicate(v, chunk_predicate), + Column::U64(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::String(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::Bool(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::Tag(v, _) => self.column_value_matches_predicate(v, chunk_predicate), @@ -545,6 +556,7 @@ impl Table { let stats = match c { Column::F64(_, stats) => Statistics::F64(stats.clone()), Column::I64(_, stats) => Statistics::I64(stats.clone()), + Column::U64(_, stats) => Statistics::U64(stats.clone()), Column::Bool(_, stats) => Statistics::Bool(stats.clone()), Column::String(_, stats) | Column::Tag(_, stats) => { Statistics::String(stats.clone()) diff --git a/packers/src/packers.rs b/packers/src/packers.rs index dbbf86a0f2..59616cdf18 100644 --- a/packers/src/packers.rs +++ b/packers/src/packers.rs @@ -20,6 +20,7 @@ use std::default::Default; pub enum Packers { Float(Packer), Integer(Packer), + UInteger(Packer), Bytes(Packer), String(Packer), Boolean(Packer), @@ -52,6 +53,7 @@ impl<'a> Packers { match self { Self::Float(p) => PackerChunker::Float(p.values.chunks(chunk_size)), Self::Integer(p) => PackerChunker::Integer(p.values.chunks(chunk_size)), + Self::UInteger(p) => PackerChunker::UInteger(p.values.chunks(chunk_size)), Self::Bytes(p) => PackerChunker::Bytes(p.values.chunks(chunk_size)), Self::String(p) => PackerChunker::String(p.values.chunks(chunk_size)), Self::Boolean(p) => PackerChunker::Boolean(p.values.chunks(chunk_size)), @@ -69,6 +71,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.reserve_exact(additional), Self::Integer(p) => p.reserve_exact(additional), + Self::UInteger(p) => p.reserve_exact(additional), Self::Bytes(p) => p.reserve_exact(additional), Self::String(p) => p.reserve_exact(additional), Self::Boolean(p) => p.reserve_exact(additional), @@ -79,6 +82,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.push_option(None), Self::Integer(p) => p.push_option(None), + Self::UInteger(p) => p.push_option(None), Self::Bytes(p) => p.push_option(None), Self::String(p) => p.push_option(None), Self::Boolean(p) => p.push_option(None), @@ -90,6 +94,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.swap(a, b), Self::Integer(p) => p.swap(a, b), + Self::UInteger(p) => p.swap(a, b), Self::Bytes(p) => p.swap(a, b), Self::String(p) => p.swap(a, b), Self::Boolean(p) => p.swap(a, b), @@ -101,6 +106,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.num_rows(), Self::Integer(p) => p.num_rows(), + Self::UInteger(p) => p.num_rows(), Self::Bytes(p) => p.num_rows(), Self::String(p) => p.num_rows(), Self::Boolean(p) => p.num_rows(), @@ -114,6 +120,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.is_null(row), Self::Integer(p) => p.is_null(row), + Self::UInteger(p) => p.is_null(row), Self::Bytes(p) => p.is_null(row), Self::String(p) => p.is_null(row), Self::Boolean(p) => p.is_null(row), @@ -124,6 +131,7 @@ impl<'a> Packers { typed_packer_accessors! { (f64_packer, f64_packer_mut, f64, Float), (i64_packer, i64_packer_mut, i64, Integer), + (u64_packer, u64_packer_mut, u64, UInteger), (bytes_packer, bytes_packer_mut, ByteArray, Bytes), (str_packer, str_packer_mut, String, String), (bool_packer, bool_packer_mut, bool, Boolean), @@ -245,6 +253,7 @@ impl std::convert::From>>> for Packers { pub enum PackerChunker<'a> { Float(Chunks<'a, Option>), Integer(Chunks<'a, Option>), + UInteger(Chunks<'a, Option>), Bytes(Chunks<'a, Option>), String(Chunks<'a, Option>), Boolean(Chunks<'a, Option>), @@ -523,6 +532,7 @@ mod test { let mut packers: Vec = Vec::new(); packers.push(Packers::Float(Packer::new())); packers.push(Packers::Integer(Packer::new())); + packers.push(Packers::UInteger(Packer::new())); packers.push(Packers::Boolean(Packer::new())); packers.get_mut(0).unwrap().f64_packer_mut().push(22.033); From abb991ffca554fa7f8c152e09e1832203f98a430 Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Mon, 8 Mar 2021 15:22:10 -0800 Subject: [PATCH 005/104] fix: add tests to unsigned integer parsing in LP and MUB --- influxdb_line_protocol/src/lib.rs | 80 ++++++++++++++++++++++++++++--- mutable_buffer/src/table.rs | 3 +- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/influxdb_line_protocol/src/lib.rs b/influxdb_line_protocol/src/lib.rs index f38d2d8f73..c586ec64c1 100644 --- a/influxdb_line_protocol/src/lib.rs +++ b/influxdb_line_protocol/src/lib.rs @@ -1053,6 +1053,13 @@ mod test { } } + fn unwrap_u64(&self) -> u64 { + match self { + Self::U64(v) => *v, + _ => panic!("field was not an u64"), + } + } + fn unwrap_f64(&self) -> f64 { match self { Self::F64(v) => *v, @@ -1218,6 +1225,19 @@ mod test { Ok(()) } + #[test] + fn parse_single_field_unteger() -> Result { + let input = "foo asdf=23u 1234"; + let vals = parse(input)?; + + assert_eq!(vals[0].series.measurement, "foo"); + assert_eq!(vals[0].timestamp, Some(1234)); + assert_eq!(vals[0].field_set[0].0, "asdf"); + assert_eq!(vals[0].field_set[0].1.unwrap_u64(), 23); + + Ok(()) + } + #[test] fn parse_single_field_float_no_decimal() -> Result { let input = "foo asdf=44 546"; @@ -1356,6 +1376,23 @@ mod test { Ok(()) } + #[test] + fn parse_two_fields_unteger() -> Result { + let input = "foo asdf=23u,bar=5u 1234"; + let vals = parse(input)?; + + assert_eq!(vals[0].series.measurement, "foo"); + assert_eq!(vals[0].timestamp, Some(1234)); + + assert_eq!(vals[0].field_set[0].0, "asdf"); + assert_eq!(vals[0].field_set[0].1.unwrap_u64(), 23); + + assert_eq!(vals[0].field_set[1].0, "bar"); + assert_eq!(vals[0].field_set[1].1.unwrap_u64(), 5); + + Ok(()) + } + #[test] fn parse_two_fields_float() -> Result { let input = "foo asdf=23.1,bar=5 1234"; @@ -1381,7 +1418,7 @@ mod test { #[test] fn parse_mixed_field_types() -> Result { - let input = r#"foo asdf=23.1,bar=5i,baz="the string",frab=false 1234"#; + let input = r#"foo asdf=23.1,bar=-5i,qux=9u,baz="the string",frab=false 1234"#; let vals = parse(input)?; assert_eq!(vals[0].series.measurement, "foo"); @@ -1394,13 +1431,16 @@ mod test { )); assert_eq!(vals[0].field_set[1].0, "bar"); - assert_eq!(vals[0].field_set[1].1.unwrap_i64(), 5); + assert_eq!(vals[0].field_set[1].1.unwrap_i64(), -5); - assert_eq!(vals[0].field_set[2].0, "baz"); - assert_eq!(vals[0].field_set[2].1.unwrap_string(), "the string"); + assert_eq!(vals[0].field_set[2].0, "qux"); + assert_eq!(vals[0].field_set[2].1.unwrap_u64(), 9); - assert_eq!(vals[0].field_set[3].0, "frab"); - assert_eq!(vals[0].field_set[3].1.unwrap_bool(), false); + assert_eq!(vals[0].field_set[3].0, "baz"); + assert_eq!(vals[0].field_set[3].1.unwrap_string(), "the string"); + + assert_eq!(vals[0].field_set[4].0, "frab"); + assert_eq!(vals[0].field_set[4].1.unwrap_bool(), false); Ok(()) } @@ -1416,6 +1456,20 @@ mod test { Ok(()) } + #[test] + fn parse_negative_uinteger() -> Result { + let input = "m0 field=-1u 99"; + let parsed = parse(input); + + assert!( + matches!(parsed, Err(super::Error::UIntegerValueInvalid { .. })), + "Wrong error: {:?}", + parsed, + ); + + Ok(()) + } + #[test] fn parse_negative_float() -> Result { let input = "m0 field2=-1 99"; @@ -1444,6 +1498,20 @@ mod test { Ok(()) } + #[test] + fn parse_out_of_range_uinteger() -> Result { + let input = "m0 field=99999999999999999999999999999999u 99"; + let parsed = parse(input); + + assert!( + matches!(parsed, Err(super::Error::UIntegerValueInvalid { .. })), + "Wrong error: {:?}", + parsed, + ); + + Ok(()) + } + #[test] fn parse_out_of_range_float() -> Result { let input = format!("m0 field={val}.{val} 99", val = "9".repeat(200)); diff --git a/mutable_buffer/src/table.rs b/mutable_buffer/src/table.rs index 35e1458dd5..aaa911649b 100644 --- a/mutable_buffer/src/table.rs +++ b/mutable_buffer/src/table.rs @@ -748,7 +748,7 @@ mod tests { let mut table = Table::new(dictionary.lookup_value_or_insert("table_name")); let lp_lines = vec![ - "h2o,state=MA,city=Boston float_field=70.4,int_field=8i,bool_field=t,string_field=\"foo\" 100", + "h2o,state=MA,city=Boston float_field=70.4,int_field=8i,uint_field=42u,bool_field=t,string_field=\"foo\" 100", ]; write_lines_to_table(&mut table, dictionary, lp_lines); @@ -760,6 +760,7 @@ mod tests { .tag("city") .field("float_field", ArrowDataType::Float64) .field("int_field", ArrowDataType::Int64) + .field("uint_field", ArrowDataType::UInt64) .tag("state") .field("string_field", ArrowDataType::Utf8) .timestamp() From 706115178b080c674e342c8c9d209eafb92ea381 Mon Sep 17 00:00:00 2001 From: Paul Dix Date: Tue, 9 Mar 2021 08:48:59 -0500 Subject: [PATCH 006/104] chore: remove cruft from config/database rules (#935) The replication, query, and subscription concepts here are going to be signficiantly different. Thought it would be best to just remove this cruft for now to avoid confusion. Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- data_types/src/database_rules.rs | 367 +----------------- .../iox/management/v1/base_types.proto | 7 - .../iox/management/v1/database_rules.proto | 93 ----- server/src/config.rs | 18 +- server/src/lib.rs | 206 +--------- tests/end_to_end_cases/management_api.rs | 21 - 6 files changed, 5 insertions(+), 707 deletions(-) diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index 70909620e4..5313564c9d 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -33,73 +33,11 @@ pub struct DatabaseRules { /// database call, so an empty default is fine. #[serde(default)] pub name: String, // TODO: Use DatabaseName here + /// Template that generates a partition key for each row inserted into the /// db #[serde(default)] pub partition_template: PartitionTemplate, - /// The set of host groups that data should be replicated to. Which host a - /// write goes to within a host group is determined by consistent hashing of - /// the partition key. We'd use this to create a host group per - /// availability zone, so you might have 5 availability zones with 2 - /// hosts in each. Replication will ensure that N of those zones get a - /// write. For each zone, only a single host needs to get the write. - /// Replication is for ensuring a write exists across multiple hosts - /// before returning success. Its purpose is to ensure write durability, - /// rather than write availability for query (this is covered by - /// subscriptions). - #[serde(default)] - pub replication: Vec, - /// The minimum number of host groups to replicate a write to before success - /// is returned. This can be overridden on a per request basis. - /// Replication will continue to write to the other host groups in the - /// background. - #[serde(default)] - pub replication_count: u8, - /// How long the replication queue can get before either rejecting writes or - /// dropping missed writes. The queue is kept in memory on a - /// per-database basis. A queue size of zero means it will only try to - /// replicate synchronously and drop any failures. - #[serde(default)] - pub replication_queue_max_size: usize, - /// `subscriptions` are used for query servers to get data via either push - /// or pull as it arrives. They are separate from replication as they - /// have a different purpose. They're for query servers or other clients - /// that want to subscribe to some subset of data being written in. This - /// could either be specific partitions, ranges of partitions, tables, or - /// rows matching some predicate. This is step #3 from the diagram. - #[serde(default)] - pub subscriptions: Vec, - - /// If set to `true`, this server should answer queries from one or more of - /// of its local write buffer and any read-only partitions that it knows - /// about. In this case, results will be merged with any others from the - /// remote goups or read-only partitions. - #[serde(default)] - pub query_local: bool, - /// Set `primary_query_group` to a host group if remote servers should be - /// issued queries for this database. All hosts in the group should be - /// queried with this server acting as the coordinator that merges - /// results together. If a specific host in the group is unavailable, - /// another host in the same position from a secondary group should be - /// queried. For example, imagine we've partitioned the data in this DB into - /// 4 partitions and we are replicating the data across 3 availability - /// zones. We have 4 hosts in each of those AZs, thus they each have 1 - /// partition. We'd set the primary group to be the 4 hosts in the same - /// AZ as this one, and the secondary groups as the hosts in the other 2 - /// AZs. - #[serde(default)] - pub primary_query_group: Option, - #[serde(default)] - pub secondary_query_groups: Vec, - - /// Use `read_only_partitions` when a server should answer queries for - /// partitions that come from object storage. This can be used to start - /// up a new query server to handle queries by pointing it at a - /// collection of partitions and then telling it to also pull - /// data from the replication servers (writes that haven't been snapshotted - /// into a partition). - #[serde(default)] - pub read_only_partitions: Vec, /// When set this will buffer WAL writes in memory based on the /// configuration. @@ -149,28 +87,9 @@ impl Partitioner for DatabaseRules { impl From for management::DatabaseRules { fn from(rules: DatabaseRules) -> Self { - let subscriptions: Vec = - rules.subscriptions.into_iter().map(Into::into).collect(); - - let replication_config = management::ReplicationConfig { - replications: rules.replication, - replication_count: rules.replication_count as _, - replication_queue_max_size: rules.replication_queue_max_size as _, - }; - - let query_config = management::QueryConfig { - query_local: rules.query_local, - primary: rules.primary_query_group.unwrap_or_default(), - secondaries: rules.secondary_query_groups, - read_only_partitions: rules.read_only_partitions, - }; - Self { name: rules.name, partition_template: Some(rules.partition_template.into()), - replication_config: Some(replication_config), - subscription_config: Some(management::SubscriptionConfig { subscriptions }), - query_config: Some(query_config), wal_buffer_config: rules.wal_buffer_config.map(Into::into), mutable_buffer_config: rules.mutable_buffer_config.map(Into::into), } @@ -183,15 +102,6 @@ impl TryFrom for DatabaseRules { fn try_from(proto: management::DatabaseRules) -> Result { DatabaseName::new(&proto.name).field("name")?; - let subscriptions = proto - .subscription_config - .map(|s| { - s.subscriptions - .vec_field("subscription_config.subscriptions") - }) - .transpose()? - .unwrap_or_default(); - let wal_buffer_config = proto.wal_buffer_config.optional("wal_buffer_config")?; let mutable_buffer_config = proto @@ -203,20 +113,9 @@ impl TryFrom for DatabaseRules { .optional("partition_template")? .unwrap_or_default(); - let query = proto.query_config.unwrap_or_default(); - let replication = proto.replication_config.unwrap_or_default(); - Ok(Self { name: proto.name, partition_template, - replication: replication.replications, - replication_count: replication.replication_count as _, - replication_queue_max_size: replication.replication_queue_max_size as _, - subscriptions, - query_local: query.query_local, - primary_query_group: query.primary.optional(), - secondary_query_groups: query.secondaries, - read_only_partitions: query.read_only_partitions, wal_buffer_config, mutable_buffer_config, }) @@ -809,123 +708,6 @@ impl TryFrom for TemplatePart { pub type PartitionId = String; pub type WriterId = u32; -/// `Subscription` represents a group of hosts that want to receive data as it -/// arrives. The subscription has a matcher that is used to determine what data -/// will match it, and an optional queue for storing matched writes. Subscribers -/// that recieve some subeset of an individual replicated write will get a new -/// replicated write, but with the same originating writer ID and sequence -/// number for the consuming subscriber's tracking purposes. -/// -/// For pull based subscriptions, the requester will send a matcher, which the -/// receiver will execute against its in-memory WAL. -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct Subscription { - pub name: String, - pub host_group_id: HostGroupId, - pub matcher: Matcher, -} - -impl From for management::subscription_config::Subscription { - fn from(s: Subscription) -> Self { - Self { - name: s.name, - host_group_id: s.host_group_id, - matcher: Some(s.matcher.into()), - } - } -} - -impl TryFrom for Subscription { - type Error = FieldViolation; - - fn try_from(proto: management::subscription_config::Subscription) -> Result { - Ok(Self { - name: proto.name.required("name")?, - host_group_id: proto.host_group_id.required("host_group_id")?, - matcher: proto.matcher.optional("matcher")?.unwrap_or_default(), - }) - } -} - -/// `Matcher` specifies the rule against the table name and/or a predicate -/// against the row to determine if it matches the write rule. -#[derive(Debug, Default, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct Matcher { - pub tables: MatchTables, - // TODO: make this work with query::Predicate - #[serde(skip_serializing_if = "Option::is_none")] - pub predicate: Option, -} - -impl From for management::Matcher { - fn from(m: Matcher) -> Self { - Self { - predicate: m.predicate.unwrap_or_default(), - table_matcher: Some(m.tables.into()), - } - } -} - -impl TryFrom for Matcher { - type Error = FieldViolation; - - fn try_from(proto: management::Matcher) -> Result { - Ok(Self { - tables: proto.table_matcher.required("table_matcher")?, - predicate: proto.predicate.optional(), - }) - } -} - -/// `MatchTables` looks at the table name of a row to determine if it should -/// match the rule. -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -#[serde(rename_all = "camelCase")] -pub enum MatchTables { - #[serde(rename = "*")] - All, - Table(String), - Regex(String), -} - -impl Default for MatchTables { - fn default() -> Self { - Self::All - } -} - -impl From for management::matcher::TableMatcher { - fn from(m: MatchTables) -> Self { - match m { - MatchTables::All => Self::All(Empty {}), - MatchTables::Table(table) => Self::Table(table), - MatchTables::Regex(regex) => Self::Regex(regex), - } - } -} - -impl TryFrom for MatchTables { - type Error = FieldViolation; - - fn try_from(proto: management::matcher::TableMatcher) -> Result { - use management::matcher::TableMatcher; - Ok(match proto { - TableMatcher::All(_) => Self::All, - TableMatcher::Table(table) => Self::Table(table.required("table_matcher.table")?), - TableMatcher::Regex(regex) => Self::Regex(regex.required("table_matcher.regex")?), - }) - } -} - -pub type HostGroupId = String; - -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct HostGroup { - pub id: HostGroupId, - /// `hosts` is a vector of connection strings for remote hosts. - pub hosts: Vec, -} - #[cfg(test)] mod tests { use influxdb_line_protocol::parse_lines; @@ -1107,16 +889,9 @@ mod tests { assert_eq!(protobuf.name, back.name); assert_eq!(rules.partition_template.parts.len(), 0); - assert_eq!(rules.subscriptions.len(), 0); - assert!(rules.primary_query_group.is_none()); - assert_eq!(rules.read_only_partitions.len(), 0); - assert_eq!(rules.secondary_query_groups.len(), 0); // These will be defaulted as optionality not preserved on non-protobuf // DatabaseRules - assert_eq!(back.replication_config, Some(Default::default())); - assert_eq!(back.subscription_config, Some(Default::default())); - assert_eq!(back.query_config, Some(Default::default())); assert_eq!(back.partition_template, Some(Default::default())); // These should be none as preserved on non-protobuf DatabaseRules @@ -1124,65 +899,6 @@ mod tests { assert!(back.mutable_buffer_config.is_none()); } - #[test] - fn test_database_rules_query() { - let readonly = vec!["readonly1".to_string(), "readonly2".to_string()]; - let secondaries = vec!["secondary1".to_string(), "secondary2".to_string()]; - - let protobuf = management::DatabaseRules { - name: "database".to_string(), - query_config: Some(management::QueryConfig { - query_local: true, - primary: "primary".to_string(), - secondaries: secondaries.clone(), - read_only_partitions: readonly.clone(), - }), - ..Default::default() - }; - - let rules: DatabaseRules = protobuf.clone().try_into().unwrap(); - let back: management::DatabaseRules = rules.clone().into(); - - assert_eq!(rules.name, protobuf.name); - assert_eq!(protobuf.name, back.name); - - assert_eq!(rules.read_only_partitions, readonly); - assert_eq!(rules.primary_query_group, Some("primary".to_string())); - assert_eq!(rules.secondary_query_groups, secondaries); - assert_eq!(rules.subscriptions.len(), 0); - assert_eq!(rules.partition_template.parts.len(), 0); - - // Should be the same as was specified - assert_eq!(back.query_config, protobuf.query_config); - assert!(back.wal_buffer_config.is_none()); - assert!(back.mutable_buffer_config.is_none()); - - // These will be defaulted as optionality not preserved on non-protobuf - // DatabaseRules - assert_eq!(back.replication_config, Some(Default::default())); - assert_eq!(back.subscription_config, Some(Default::default())); - assert_eq!(back.partition_template, Some(Default::default())); - } - - #[test] - fn test_query_config_default() { - let protobuf = management::DatabaseRules { - name: "database".to_string(), - query_config: Some(Default::default()), - ..Default::default() - }; - - let rules: DatabaseRules = protobuf.clone().try_into().unwrap(); - let back: management::DatabaseRules = rules.clone().into(); - - assert!(rules.primary_query_group.is_none()); - assert_eq!(rules.secondary_query_groups.len(), 0); - assert_eq!(rules.read_only_partitions.len(), 0); - assert_eq!(rules.query_local, false); - - assert_eq!(protobuf.query_config, back.query_config); - } - #[test] fn test_partition_template_default() { let protobuf = management::DatabaseRules { @@ -1317,87 +1033,6 @@ mod tests { assert_eq!(&err.description, "Duration must be positive"); } - #[test] - fn test_matcher_default() { - let protobuf: management::Matcher = Default::default(); - - let res: Result = protobuf.try_into(); - let err = res.expect_err("expected failure"); - - assert_eq!(&err.field, "table_matcher"); - assert_eq!(&err.description, "Field is required"); - } - - #[test] - fn test_matcher() { - let protobuf = management::Matcher { - predicate: Default::default(), - table_matcher: Some(management::matcher::TableMatcher::Regex( - "regex".to_string(), - )), - }; - let matcher: Matcher = protobuf.try_into().unwrap(); - - assert_eq!(matcher.tables, MatchTables::Regex("regex".to_string())); - assert!(matcher.predicate.is_none()); - } - - #[test] - fn test_subscription_default() { - let pb_matcher = Some(management::Matcher { - predicate: "predicate1".to_string(), - table_matcher: Some(management::matcher::TableMatcher::Table( - "table".to_string(), - )), - }); - - let matcher = Matcher { - tables: MatchTables::Table("table".to_string()), - predicate: Some("predicate1".to_string()), - }; - - let subscription_config = management::SubscriptionConfig { - subscriptions: vec![ - management::subscription_config::Subscription { - name: "subscription1".to_string(), - host_group_id: "host group".to_string(), - matcher: pb_matcher.clone(), - }, - management::subscription_config::Subscription { - name: "subscription2".to_string(), - host_group_id: "host group".to_string(), - matcher: pb_matcher, - }, - ], - }; - - let protobuf = management::DatabaseRules { - name: "database".to_string(), - subscription_config: Some(subscription_config), - ..Default::default() - }; - - let rules: DatabaseRules = protobuf.clone().try_into().unwrap(); - let back: management::DatabaseRules = rules.clone().into(); - - assert_eq!(protobuf.subscription_config, back.subscription_config); - assert_eq!( - rules.subscriptions, - vec![ - Subscription { - name: "subscription1".to_string(), - host_group_id: "host group".to_string(), - matcher: matcher.clone() - }, - Subscription { - name: "subscription2".to_string(), - host_group_id: "host group".to_string(), - matcher - } - ] - ) - } - #[test] fn mutable_buffer_config_default() { let protobuf: management::MutableBufferConfig = Default::default(); diff --git a/generated_types/protos/influxdata/iox/management/v1/base_types.proto b/generated_types/protos/influxdata/iox/management/v1/base_types.proto index 90398b778a..53b9dc73d2 100644 --- a/generated_types/protos/influxdata/iox/management/v1/base_types.proto +++ b/generated_types/protos/influxdata/iox/management/v1/base_types.proto @@ -21,10 +21,3 @@ enum ColumnType { COLUMN_TYPE_STRING = 4; COLUMN_TYPE_BOOL = 5; } - -message HostGroup { - string id = 1; - - // connection strings for remote hosts. - repeated string hosts = 2; -} diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto index df953f4224..3a7fd2c566 100644 --- a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto +++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto @@ -31,90 +31,6 @@ message PartitionTemplate { repeated Part parts = 1; } -message Matcher { - // A query predicate to filter rows - string predicate = 1; - // Restrict selection to a specific table or tables specified by a regex - oneof table_matcher { - google.protobuf.Empty all = 2; - string table = 3; - string regex = 4; - } -} - -message ReplicationConfig { - // The set of host groups that data should be replicated to. Which host a - // write goes to within a host group is determined by consistent hashing of - // the partition key. We'd use this to create a host group per - // availability zone, so you might have 5 availability zones with 2 - // hosts in each. Replication will ensure that N of those zones get a - // write. For each zone, only a single host needs to get the write. - // Replication is for ensuring a write exists across multiple hosts - // before returning success. Its purpose is to ensure write durability, - // rather than write availability for query (this is covered by - // subscriptions). - repeated string replications = 1; - - // The minimum number of host groups to replicate a write to before success - // is returned. This can be overridden on a per request basis. - // Replication will continue to write to the other host groups in the - // background. - uint32 replication_count = 2; - - // How long the replication queue can get before either rejecting writes or - // dropping missed writes. The queue is kept in memory on a - // per-database basis. A queue size of zero means it will only try to - // replicate synchronously and drop any failures. - uint64 replication_queue_max_size = 3; -} - -message SubscriptionConfig { - message Subscription { - string name = 1; - string host_group_id = 2; - Matcher matcher = 3; - } - - // `subscriptions` are used for query servers to get data via either push - // or pull as it arrives. They are separate from replication as they - // have a different purpose. They're for query servers or other clients - // that want to subscribe to some subset of data being written in. This - // could either be specific partitions, ranges of partitions, tables, or - // rows matching some predicate. - repeated Subscription subscriptions = 1; -} - -message QueryConfig { - // If set to `true`, this server should answer queries from one or more of - // of its local write buffer and any read-only partitions that it knows - // about. In this case, results will be merged with any others from the - // remote goups or read-only partitions. - bool query_local = 1; - - // Set `primary` to a host group if remote servers should be - // issued queries for this database. All hosts in the group should be - // queried with this server acting as the coordinator that merges - // results together. - string primary = 2; - - // If a specific host in the primary group is unavailable, - // another host in the same position from a secondary group should be - // queried. For example, imagine we've partitioned the data in this DB into - // 4 partitions and we are replicating the data across 3 availability - // zones. We have 4 hosts in each of those AZs, thus they each have 1 - // partition. We'd set the primary group to be the 4 hosts in the same - // AZ as this one, and the secondary groups as the hosts in the other 2 AZs. - repeated string secondaries = 3; - - // Use `readOnlyPartitions` when a server should answer queries for - // partitions that come from object storage. This can be used to start - // up a new query server to handle queries by pointing it at a - // collection of partitions and then telling it to also pull - // data from the replication servers (writes that haven't been snapshotted - // into a partition). - repeated string read_only_partitions = 4; -} - message WalBufferConfig { enum Rollover { ROLLOVER_UNSPECIFIED = 0; @@ -231,15 +147,6 @@ message DatabaseRules { // Template that generates a partition key for each row inserted into the database PartitionTemplate partition_template = 2; - // Synchronous replication configuration for this database - ReplicationConfig replication_config = 3; - - // Asynchronous pull-based subscription configuration for this database - SubscriptionConfig subscription_config = 4; - - // Query configuration for this database - QueryConfig query_config = 5; - // WAL configuration for this database WalBufferConfig wal_buffer_config = 6; diff --git a/server/src/config.rs b/server/src/config.rs index b7781734ac..7c702af1f4 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -1,9 +1,6 @@ /// This module contains code for managing the configuration of the server. use crate::{db::Db, Error, Result}; -use data_types::{ - database_rules::{DatabaseRules, HostGroup, HostGroupId}, - DatabaseName, -}; +use data_types::{database_rules::DatabaseRules, DatabaseName}; use mutable_buffer::MutableBufferDb; use object_store::path::ObjectStorePath; use read_buffer::Database as ReadBufferDb; @@ -60,18 +57,6 @@ impl Config { state.databases.get(name).cloned() } - pub(crate) fn create_host_group(&self, host_group: HostGroup) { - let mut state = self.state.write().expect("mutex poisoned"); - state - .host_groups - .insert(host_group.id.clone(), Arc::new(host_group)); - } - - pub(crate) fn host_group(&self, host_group_id: &str) -> Option> { - let state = self.state.read().expect("mutex poisoned"); - state.host_groups.get(host_group_id).cloned() - } - pub(crate) fn db_names_sorted(&self) -> Vec> { let state = self.state.read().expect("mutex poisoned"); state.databases.keys().cloned().collect() @@ -106,7 +91,6 @@ pub fn object_store_path_for_database_config( struct ConfigState { reservations: BTreeSet>, databases: BTreeMap, Arc>, - host_groups: BTreeMap>, } /// CreateDatabaseHandle is retunred when a call is made to `create_db` on diff --git a/server/src/lib.rs b/server/src/lib.rs index 33ca685920..06fab7c3a0 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -89,7 +89,7 @@ use crate::{ }; use data_types::{ data::{lines_to_replicated_write, ReplicatedWrite}, - database_rules::{DatabaseRules, HostGroup, HostGroupId, MatchTables}, + database_rules::DatabaseRules, {DatabaseName, DatabaseNameError}, }; use influxdb_line_protocol::ParsedLine; @@ -119,10 +119,6 @@ pub enum Error { UnknownDatabaseError { source: DatabaseError }, #[snafu(display("no local buffer for database: {}", db))] NoLocalBuffer { db: String }, - #[snafu(display("host group not found: {}", id))] - HostGroupNotFound { id: HostGroupId }, - #[snafu(display("no hosts in group: {}", id))] - NoHostInGroup { id: HostGroupId }, #[snafu(display("unable to get connection to remote server: {}", server))] UnableToGetConnection { server: String, @@ -294,18 +290,6 @@ impl Server { Ok(()) } - /// Creates a host group with a set of connection strings to hosts. These - /// host connection strings should be something that the connection - /// manager can use to return a remote server to work with. - pub async fn create_host_group(&mut self, id: HostGroupId, hosts: Vec) -> Result<()> { - // Return an error if this server hasn't yet been setup with an id - self.require_id()?; - - self.config.create_host_group(HostGroup { id, hosts }); - - Ok(()) - } - /// `write_lines` takes in raw line protocol and converts it to a /// `ReplicatedWrite`, which is then replicated to other servers based /// on the configuration of the `db`. This is step #1 from the crate @@ -373,59 +357,6 @@ impl Server { } } - for host_group_id in &db.rules.replication { - self.replicate_to_host_group(host_group_id, db_name, &write) - .await?; - } - - for subscription in &db.rules.subscriptions { - match subscription.matcher.tables { - MatchTables::All => { - self.replicate_to_host_group(&subscription.host_group_id, db_name, &write) - .await? - } - MatchTables::Table(_) => unimplemented!(), - MatchTables::Regex(_) => unimplemented!(), - } - } - - Ok(()) - } - - // replicates to a single host in the group based on hashing rules. If that host - // is unavailable an error will be returned. The request may still succeed - // if enough of the other host groups have returned a success. - async fn replicate_to_host_group( - &self, - host_group_id: &str, - db_name: &DatabaseName<'_>, - write: &ReplicatedWrite, - ) -> Result<()> { - let group = self - .config - .host_group(host_group_id) - .context(HostGroupNotFound { id: host_group_id })?; - - // TODO: handle hashing rules to determine which host in the group should get - // the write. for now, just write to the first one. - let host = group - .hosts - .get(0) - .context(NoHostInGroup { id: host_group_id })?; - - let connection = self - .connection_manager - .remote_server(host) - .await - .map_err(|e| Box::new(e) as DatabaseError) - .context(UnableToGetConnection { server: host })?; - - connection - .replicate(db_name, write) - .await - .map_err(|e| Box::new(e) as DatabaseError) - .context(ErrorReplicating {})?; - Ok(()) } @@ -567,8 +498,7 @@ mod tests { use arrow_deps::{assert_table_eq, datafusion::physical_plan::collect}; use async_trait::async_trait; use data_types::database_rules::{ - MatchTables, Matcher, PartitionTemplate, Subscription, TemplatePart, WalBufferConfig, - WalBufferRollover, + PartitionTemplate, TemplatePart, WalBufferConfig, WalBufferRollover, }; use futures::TryStreamExt; use influxdb_line_protocol::parse_lines; @@ -585,7 +515,7 @@ mod tests { async fn server_api_calls_return_error_with_no_id_set() -> Result { let manager = TestConnectionManager::new(); let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); - let mut server = Server::new(manager, store); + let server = Server::new(manager, store); let rules = DatabaseRules::new(); let resp = server.create_database("foo", rules).await.unwrap_err(); @@ -595,12 +525,6 @@ mod tests { let resp = server.write_lines("foo", &lines).await.unwrap_err(); assert!(matches!(resp, Error::IdNotSet)); - let resp = server - .create_host_group("group1".to_string(), vec!["serverA".to_string()]) - .await - .unwrap_err(); - assert!(matches!(resp, Error::IdNotSet)); - Ok(()) } @@ -776,130 +700,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn replicate_to_single_group() -> Result { - let mut manager = TestConnectionManager::new(); - let remote = Arc::new(TestRemoteServer::default()); - let remote_id = "serverA"; - manager - .remotes - .insert(remote_id.to_string(), Arc::clone(&remote)); - - let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); - - let mut server = Server::new(manager, store); - server.set_id(1); - let host_group_id = "az1".to_string(); - let rules = DatabaseRules { - replication: vec![host_group_id.clone()], - replication_count: 1, - ..Default::default() - }; - server - .create_host_group(host_group_id.clone(), vec![remote_id.to_string()]) - .await - .unwrap(); - let db_name = "foo"; - server.create_database(db_name, rules).await.unwrap(); - - let lines = parsed_lines("cpu bar=1 10"); - server.write_lines("foo", &lines).await.unwrap(); - - let writes = remote.writes.lock().get(db_name).unwrap().clone(); - - let write_text = r#" -writer:1, sequence:1, checksum:226387645 -partition_key: - table:cpu - bar:1 time:10 -"#; - - assert_eq!(write_text, writes[0].to_string()); - - // ensure sequence number goes up - let lines = parsed_lines("mem,server=A,region=west user=232 12"); - server.write_lines("foo", &lines).await.unwrap(); - - let writes = remote.writes.lock().get(db_name).unwrap().clone(); - assert_eq!(2, writes.len()); - - let write_text = r#" -writer:1, sequence:2, checksum:3759030699 -partition_key: - table:mem - server:A region:west user:232 time:12 -"#; - - assert_eq!(write_text, writes[1].to_string()); - - Ok(()) - } - - #[tokio::test] - async fn sends_all_to_subscriber() -> Result { - let mut manager = TestConnectionManager::new(); - let remote = Arc::new(TestRemoteServer::default()); - let remote_id = "serverA"; - manager - .remotes - .insert(remote_id.to_string(), Arc::clone(&remote)); - - let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); - - let mut server = Server::new(manager, store); - server.set_id(1); - let host_group_id = "az1".to_string(); - let rules = DatabaseRules { - subscriptions: vec![Subscription { - name: "query_server_1".to_string(), - host_group_id: host_group_id.clone(), - matcher: Matcher { - tables: MatchTables::All, - predicate: None, - }, - }], - ..Default::default() - }; - server - .create_host_group(host_group_id.clone(), vec![remote_id.to_string()]) - .await - .unwrap(); - let db_name = "foo"; - server.create_database(db_name, rules).await.unwrap(); - - let lines = parsed_lines("cpu bar=1 10"); - server.write_lines("foo", &lines).await.unwrap(); - - let writes = remote.writes.lock().get(db_name).unwrap().clone(); - - let write_text = r#" -writer:1, sequence:1, checksum:226387645 -partition_key: - table:cpu - bar:1 time:10 -"#; - - assert_eq!(write_text, writes[0].to_string()); - - // ensure sequence number goes up - let lines = parsed_lines("mem,server=A,region=west user=232 12"); - server.write_lines("foo", &lines).await.unwrap(); - - let writes = remote.writes.lock().get(db_name).unwrap().clone(); - assert_eq!(2, writes.len()); - - let write_text = r#" -writer:1, sequence:2, checksum:3759030699 -partition_key: - table:mem - server:A region:west user:232 time:12 -"#; - - assert_eq!(write_text, writes[1].to_string()); - - Ok(()) - } - #[tokio::test] async fn segment_persisted_on_rollover() { let manager = TestConnectionManager::new(); diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 0cc9145e62..802f0a36e8 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -93,27 +93,6 @@ async fn test_create_get_database(client: &mut Client) { part: Some(partition_template::part::Part::Table(Empty {})), }], }), - replication_config: Some(ReplicationConfig { - replications: vec!["cupcakes".to_string()], - replication_count: 3, - replication_queue_max_size: 20, - }), - subscription_config: Some(SubscriptionConfig { - subscriptions: vec![subscription_config::Subscription { - name: "subscription".to_string(), - host_group_id: "hostgroup".to_string(), - matcher: Some(Matcher { - predicate: "pred".to_string(), - table_matcher: Some(matcher::TableMatcher::All(Empty {})), - }), - }], - }), - query_config: Some(QueryConfig { - query_local: true, - primary: Default::default(), - secondaries: vec![], - read_only_partitions: vec![], - }), wal_buffer_config: Some(WalBufferConfig { buffer_size: 24, segment_size: 2, From 255589cfcbfa2323fab4112699ab7b27e3ff4553 Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Tue, 9 Mar 2021 07:29:01 -0800 Subject: [PATCH 007/104] fix: test error type --- influxdb_line_protocol/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb_line_protocol/src/lib.rs b/influxdb_line_protocol/src/lib.rs index c586ec64c1..9ede7fd3f0 100644 --- a/influxdb_line_protocol/src/lib.rs +++ b/influxdb_line_protocol/src/lib.rs @@ -1462,7 +1462,7 @@ mod test { let parsed = parse(input); assert!( - matches!(parsed, Err(super::Error::UIntegerValueInvalid { .. })), + matches!(parsed, Err(super::Error::CannotParseEntireLine { .. })), "Wrong error: {:?}", parsed, ); From 09c4f88758ed3448cd34ab6ee66fb6286a689436 Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Tue, 9 Mar 2021 07:45:05 -0800 Subject: [PATCH 008/104] fix: cargo fmt --- mutable_buffer/src/table.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mutable_buffer/src/table.rs b/mutable_buffer/src/table.rs index aaa911649b..5a2833e560 100644 --- a/mutable_buffer/src/table.rs +++ b/mutable_buffer/src/table.rs @@ -23,7 +23,9 @@ use snafu::{OptionExt, ResultExt, Snafu}; use arrow_deps::{ arrow, arrow::{ - array::{ArrayRef, BooleanBuilder, Float64Builder, Int64Builder, UInt64Builder, StringBuilder}, + array::{ + ArrayRef, BooleanBuilder, Float64Builder, Int64Builder, StringBuilder, UInt64Builder, + }, datatypes::DataType as ArrowDataType, record_batch::RecordBatch, }, From 488252d9f106497f1805a2e2638f1df5c6b39e05 Mon Sep 17 00:00:00 2001 From: Jacob Marble Date: Tue, 9 Mar 2021 07:52:52 -0800 Subject: [PATCH 009/104] fix: test --- mutable_buffer/src/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutable_buffer/src/table.rs b/mutable_buffer/src/table.rs index 5a2833e560..43d275ab6f 100644 --- a/mutable_buffer/src/table.rs +++ b/mutable_buffer/src/table.rs @@ -762,10 +762,10 @@ mod tests { .tag("city") .field("float_field", ArrowDataType::Float64) .field("int_field", ArrowDataType::Int64) - .field("uint_field", ArrowDataType::UInt64) .tag("state") .field("string_field", ArrowDataType::Utf8) .timestamp() + .field("uint_field", ArrowDataType::UInt64) .build() .unwrap(); From 1af5cf8b7ca630d3a2cf45aa064cdbee159c302d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 9 Mar 2021 14:08:55 -0500 Subject: [PATCH 010/104] refactor: Move end-to-end test server fixture into its own module (#945) * refactor: Move test server fixture into its own module * fix: Update tests/end-to-end.rs * fix: better error handling and display * fix: tweak startup message --- Cargo.lock | 1 + Cargo.toml | 1 + tests/common/mod.rs | 1 + tests/common/server_fixture.rs | 291 ++++++++++++++++++++++++++++ tests/end-to-end.rs | 156 +-------------- tests/end_to_end_cases/read_api.rs | 31 ++- tests/end_to_end_cases/write_api.rs | 14 +- 7 files changed, 320 insertions(+), 175 deletions(-) create mode 100644 tests/common/mod.rs create mode 100644 tests/common/server_fixture.rs diff --git a/Cargo.lock b/Cargo.lock index a2903ca1ab..35d582dd7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1502,6 +1502,7 @@ dependencies = [ "mem_qe", "mutable_buffer", "object_store", + "once_cell", "opentelemetry", "opentelemetry-jaeger", "packers", diff --git a/Cargo.toml b/Cargo.toml index cb6c61d6ba..3c0f2f75ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,6 +93,7 @@ tracing-subscriber = { version = "0.2.15", features = ["parking_lot"] } influxdb2_client = { path = "influxdb2_client" } influxdb_iox_client = { path = "influxdb_iox_client", features = ["flight"] } test_helpers = { path = "test_helpers" } +once_cell = { version = "1.4.0", features = ["parking_lot"] } # Crates.io dependencies, in alphabetical order assert_cmd = "1.0.0" diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000000..a057eb4940 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1 @@ +pub mod server_fixture; diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs new file mode 100644 index 0000000000..497f45cbcd --- /dev/null +++ b/tests/common/server_fixture.rs @@ -0,0 +1,291 @@ +use std::{fs::File, str}; +use std::{ + num::NonZeroU32, + process::{Child, Command}, +}; + +use assert_cmd::prelude::*; +use futures::prelude::*; +use std::time::Duration; +use tempfile::TempDir; + +type Error = Box; +type Result = std::result::Result; + +// These port numbers are chosen to not collide with a development ioxd server +// running locally. +// TODO(786): allocate random free ports instead of hardcoding. +// TODO(785): we cannot use localhost here. +macro_rules! http_bind_addr { + () => { + "127.0.0.1:8090" + }; +} +macro_rules! grpc_bind_addr { + () => { + "127.0.0.1:8092" + }; +} + +const HTTP_BIND_ADDR: &str = http_bind_addr!(); +const GRPC_BIND_ADDR: &str = grpc_bind_addr!(); + +const HTTP_BASE: &str = concat!("http://", http_bind_addr!()); +const IOX_API_V1_BASE: &str = concat!("http://", http_bind_addr!(), "/iox/api/v1"); +pub const GRPC_URL_BASE: &str = concat!("http://", grpc_bind_addr!(), "/"); + +pub const TOKEN: &str = "InfluxDB IOx doesn't have authentication yet"; + +use std::sync::Arc; + +use once_cell::sync::OnceCell; +use tokio::sync::Mutex; + +/// Represents a server that has been started and is available for +/// testing. +pub struct ServerFixture { + server: Arc, +} + +impl ServerFixture { + /// Create a new server fixture and wait for it to be ready. This + /// is called "create" rather than new because it is async and waits + /// + /// This is currently implemented as a singleton so all tests *must* + /// use a new database and not interfere with the existing database. + pub async fn create_shared() -> Self { + static SERVER: OnceCell> = OnceCell::new(); + + let server = Arc::clone(SERVER.get_or_init(|| { + let server = TestServer::new().expect("Could start test server"); + Arc::new(server) + })); + + // ensure the server is ready + server.wait_until_ready().await; + ServerFixture { server } + } + + /// Return a channel connected to the gRPC API. Panics if the + /// server is not yet up + pub async fn grpc_channel(&self) -> tonic::transport::Channel { + self.server + .grpc_channel() + .await + .expect("The server should be up") + } + + /// Return the http base URL for the HTTP API + pub fn http_base(&self) -> &str { + self.server.http_base() + } + + /// Return the base URL for the IOx V1 API + pub fn iox_api_v1_base(&self) -> &str { + self.server.iox_api_v1_base() + } +} + +#[derive(Debug)] +/// Represents the current known state of a TestServer +enum ServerState { + Started, + Ready, + Error, +} + +struct TestServer { + /// Is the server ready to accept connections? + ready: Mutex, + + server_process: Child, + + // The temporary directory **must** be last so that it is + // dropped after the database closes. + dir: TempDir, +} + +impl TestServer { + fn new() -> Result { + let ready = Mutex::new(ServerState::Started); + + let dir = test_helpers::tmp_dir().unwrap(); + // Make a log file in the temporary dir (don't auto delete it to help debugging + // efforts) + let mut log_path = std::env::temp_dir(); + log_path.push("server_fixture.log"); + + println!("****************"); + println!("Server Logging to {:?}", log_path); + println!("****************"); + let log_file = File::create(log_path).expect("Opening log file"); + + let stdout_log_file = log_file + .try_clone() + .expect("cloning file handle for stdout"); + let stderr_log_file = log_file; + + let server_process = Command::cargo_bin("influxdb_iox") + .unwrap() + // Can enable for debbugging + //.arg("-vv") + .env("INFLUXDB_IOX_ID", "1") + .env("INFLUXDB_IOX_BIND_ADDR", HTTP_BIND_ADDR) + .env("INFLUXDB_IOX_GRPC_BIND_ADDR", GRPC_BIND_ADDR) + // redirect output to log file + .stdout(stdout_log_file) + .stderr(stderr_log_file) + .spawn() + .unwrap(); + + Ok(Self { + ready, + dir, + server_process, + }) + } + + #[allow(dead_code)] + fn restart(&mut self) -> Result<()> { + self.server_process.kill().unwrap(); + self.server_process.wait().unwrap(); + self.server_process = Command::cargo_bin("influxdb_iox") + .unwrap() + // Can enable for debbugging + //.arg("-vv") + .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) + .env("INFLUXDB_IOX_ID", "1") + .spawn() + .unwrap(); + Ok(()) + } + + async fn wait_until_ready(&self) { + let mut ready = self.ready.lock().await; + match *ready { + ServerState::Started => {} // first time, need to try and start it + ServerState::Ready => { + return; + } + ServerState::Error => { + panic!("Server was previously found to be in Error, aborting"); + } + } + + // Poll the RPC and HTTP servers separately as they listen on + // different ports but both need to be up for the test to run + let try_grpc_connect = async { + let mut interval = tokio::time::interval(Duration::from_millis(500)); + + loop { + match self.grpc_channel().await { + Ok(channel) => { + println!("Successfully connected to server"); + + let mut health = influxdb_iox_client::health::Client::new(channel); + + match health.check_storage().await { + Ok(_) => { + println!("Storage service is running"); + return; + } + Err(e) => { + println!("Error checking storage service status: {}", e); + } + } + } + Err(e) => { + println!("Waiting for gRPC API to be up: {}", e); + } + } + interval.tick().await; + } + }; + + let try_http_connect = async { + let client = reqwest::Client::new(); + let url = format!("{}/health", HTTP_BASE); + let mut interval = tokio::time::interval(Duration::from_millis(500)); + loop { + match client.get(&url).send().await { + Ok(resp) => { + println!("Successfully got a response from HTTP: {:?}", resp); + return; + } + Err(e) => { + println!("Waiting for HTTP server to be up: {}", e); + } + } + interval.tick().await; + } + }; + + let pair = future::join(try_http_connect, try_grpc_connect); + + let capped_check = tokio::time::timeout(Duration::from_secs(3), pair); + + match capped_check.await { + Ok(_) => { + println!("Successfully started {}", self); + *ready = ServerState::Ready; + } + Err(e) => { + // tell others that this server had some problem + *ready = ServerState::Error; + std::mem::drop(ready); + panic!("Server was not ready in required time: {}", e); + } + } + + // Set the writer id, if requested (TODO if requested) + let channel = self.grpc_channel().await.expect("gRPC should be running"); + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + let id = NonZeroU32::new(42).expect("42 is non zero, among its other properties"); + + management_client + .update_writer_id(id) + .await + .expect("set ID failed"); + } + + /// Create a connection channel for the gRPR endpoing + async fn grpc_channel( + &self, + ) -> influxdb_iox_client::connection::Result { + influxdb_iox_client::connection::Builder::default() + .build(self.grpc_url_base()) + .await + } + + fn grpc_url_base(&self) -> &str { + GRPC_URL_BASE + } + + fn http_base(&self) -> &str { + HTTP_BASE + } + + fn iox_api_v1_base(&self) -> &str { + IOX_API_V1_BASE + } +} + +impl std::fmt::Display for TestServer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + write!( + f, + "TestServer (grpc {}, http {})", + self.grpc_url_base(), + self.http_base() + ) + } +} + +impl Drop for TestServer { + fn drop(&mut self) { + self.server_process + .kill() + .expect("Should have been able to kill the test server"); + } +} diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index 4125807097..9d864966f2 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -18,15 +18,12 @@ // - Stopping the server after all relevant tests are run use std::convert::TryInto; -use std::process::{Child, Command}; use std::str; -use std::time::{Duration, SystemTime}; +use std::time::SystemTime; use std::u32; -use assert_cmd::prelude::*; use futures::prelude::*; use prost::Message; -use tempfile::TempDir; use data_types::{names::org_and_bucket_to_database, DatabaseName}; use end_to_end_cases::*; @@ -35,42 +32,19 @@ use generated_types::{ TimestampRange, }; -// These port numbers are chosen to not collide with a development ioxd server -// running locally. -// TODO(786): allocate random free ports instead of hardcoding. -// TODO(785): we cannot use localhost here. -macro_rules! http_bind_addr { - () => { - "127.0.0.1:8090" - }; -} -macro_rules! grpc_bind_addr { - () => { - "127.0.0.1:8092" - }; -} - -const HTTP_BIND_ADDR: &str = http_bind_addr!(); -const GRPC_BIND_ADDR: &str = grpc_bind_addr!(); - -const HTTP_BASE: &str = concat!("http://", http_bind_addr!()); -const IOX_API_V1_BASE: &str = concat!("http://", http_bind_addr!(), "/iox/api/v1"); -const GRPC_URL_BASE: &str = concat!("http://", grpc_bind_addr!(), "/"); - -const TOKEN: &str = "InfluxDB IOx doesn't have authentication yet"; - type Error = Box; type Result = std::result::Result; +pub mod common; mod end_to_end_cases; +use common::server_fixture::*; + #[tokio::test] async fn read_and_write_data() { - let server = TestServer::new().unwrap(); - server.wait_until_ready().await; + let fixture = ServerFixture::create_shared().await; - let http_client = reqwest::Client::new(); - let influxdb2 = influxdb2_client::Client::new(HTTP_BASE, TOKEN); + let influxdb2 = influxdb2_client::Client::new(fixture.http_base(), TOKEN); let grpc = influxdb_iox_client::connection::Builder::default() .build(GRPC_URL_BASE) .await @@ -89,7 +63,7 @@ async fn read_and_write_data() { let expected_read_data = load_data(&influxdb2, &scenario).await; let sql_query = "select * from cpu_load_short"; - read_api::test(&http_client, &scenario, sql_query, &expected_read_data).await; + read_api::test(&fixture, &scenario, sql_query, &expected_read_data).await; storage_api::test(&mut storage_client, &scenario).await; flight_api::test(&scenario, sql_query, &expected_read_data).await; } @@ -104,7 +78,6 @@ async fn read_and_write_data() { .await; management_api::test(&mut management_client).await; management_cli::test(GRPC_URL_BASE).await; - write_api::test(grpc).await; write_cli::test(GRPC_URL_BASE).await; test_http_error_messages(&influxdb2).await.unwrap(); } @@ -345,118 +318,3 @@ fn substitute_nanos(ns_since_epoch: i64, lines: &[&str]) -> Vec { }) .collect() } - -struct TestServer { - server_process: Child, - - // The temporary directory **must** be last so that it is - // dropped after the database closes. - #[allow(dead_code)] - dir: TempDir, -} - -impl TestServer { - fn new() -> Result { - let dir = test_helpers::tmp_dir().unwrap(); - - let server_process = Command::cargo_bin("influxdb_iox") - .unwrap() - // Can enable for debbugging - //.arg("-vv") - .env("INFLUXDB_IOX_ID", "1") - .env("INFLUXDB_IOX_BIND_ADDR", HTTP_BIND_ADDR) - .env("INFLUXDB_IOX_GRPC_BIND_ADDR", GRPC_BIND_ADDR) - .spawn() - .unwrap(); - - Ok(Self { - dir, - server_process, - }) - } - - #[allow(dead_code)] - fn restart(&mut self) -> Result<()> { - self.server_process.kill().unwrap(); - self.server_process.wait().unwrap(); - self.server_process = Command::cargo_bin("influxdb_iox") - .unwrap() - // Can enable for debbugging - //.arg("-vv") - .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) - .env("INFLUXDB_IOX_ID", "1") - .spawn() - .unwrap(); - Ok(()) - } - - async fn wait_until_ready(&self) { - // Poll the RPC and HTTP servers separately as they listen on - // different ports but both need to be up for the test to run - let try_grpc_connect = async { - let mut interval = tokio::time::interval(Duration::from_millis(500)); - - loop { - match influxdb_iox_client::connection::Builder::default() - .build(GRPC_URL_BASE) - .await - { - Ok(connection) => { - println!("Successfully connected to server"); - - let mut health = influxdb_iox_client::health::Client::new(connection); - - match health.check_storage().await { - Ok(_) => { - println!("Storage service is running"); - return; - } - Err(e) => { - println!("Error checking storage service status: {}", e); - } - } - } - Err(e) => { - println!("Waiting for gRPC API to be up: {}", e); - } - } - interval.tick().await; - } - }; - - let try_http_connect = async { - let client = reqwest::Client::new(); - let url = format!("{}/health", HTTP_BASE); - let mut interval = tokio::time::interval(Duration::from_millis(500)); - loop { - match client.get(&url).send().await { - Ok(resp) => { - println!("Successfully got a response from HTTP: {:?}", resp); - return; - } - Err(e) => { - println!("Waiting for HTTP server to be up: {}", e); - } - } - interval.tick().await; - } - }; - - let pair = future::join(try_http_connect, try_grpc_connect); - - let capped_check = tokio::time::timeout(Duration::from_secs(3), pair); - - match capped_check.await { - Ok(_) => println!("Server is up correctly"), - Err(e) => println!("WARNING: server was not ready: {}", e), - } - } -} - -impl Drop for TestServer { - fn drop(&mut self) { - self.server_process - .kill() - .expect("Should have been able to kill the test server"); - } -} diff --git a/tests/end_to_end_cases/read_api.rs b/tests/end_to_end_cases/read_api.rs index 8a05c1f062..b055812ff8 100644 --- a/tests/end_to_end_cases/read_api.rs +++ b/tests/end_to_end_cases/read_api.rs @@ -1,29 +1,17 @@ -use crate::{Scenario, IOX_API_V1_BASE}; +use crate::common::server_fixture::ServerFixture; +use crate::Scenario; pub async fn test( - client: &reqwest::Client, + server_fixture: &ServerFixture, scenario: &Scenario, sql_query: &str, expected_read_data: &[String], ) { - let text = read_data_as_sql(&client, scenario, sql_query).await; - - assert_eq!( - text, expected_read_data, - "Actual:\n{:#?}\nExpected:\n{:#?}", - text, expected_read_data - ); -} - -async fn read_data_as_sql( - client: &reqwest::Client, - scenario: &Scenario, - sql_query: &str, -) -> Vec { + let client = reqwest::Client::new(); let db_name = format!("{}_{}", scenario.org_id_str(), scenario.bucket_id_str()); let path = format!("/databases/{}/query", db_name); - let url = format!("{}{}", IOX_API_V1_BASE, path); - let lines = client + let url = format!("{}{}", server_fixture.iox_api_v1_base(), path); + let lines: Vec<_> = client .get(&url) .query(&[("q", sql_query)]) .send() @@ -36,5 +24,10 @@ async fn read_data_as_sql( .split('\n') .map(str::to_string) .collect(); - lines + + assert_eq!( + lines, expected_read_data, + "Actual:\n{:#?}\nExpected:\n{:#?}", + lines, expected_read_data + ); } diff --git a/tests/end_to_end_cases/write_api.rs b/tests/end_to_end_cases/write_api.rs index 49250767cb..89a51c5ac4 100644 --- a/tests/end_to_end_cases/write_api.rs +++ b/tests/end_to_end_cases/write_api.rs @@ -6,15 +6,15 @@ use test_helpers::assert_contains; use super::util::rand_name; -/// Tests the basics of the write API -pub async fn test(channel: tonic::transport::Channel) { - let mut management_client = management::Client::new(channel.clone()); - let mut write_client = write::Client::new(channel); +use crate::common::server_fixture::ServerFixture; - test_write(&mut management_client, &mut write_client).await -} +#[tokio::test] +async fn test_write() { + // TODO sort out changing the test ID + let fixture = ServerFixture::create_shared().await; + let mut management_client = management::Client::new(fixture.grpc_channel().await); + let mut write_client = write::Client::new(fixture.grpc_channel().await); -async fn test_write(management_client: &mut management::Client, write_client: &mut write::Client) { const TEST_ID: u32 = 42; let db_name = rand_name(); From 8163a31552ffe24feae0f034bb50eb2de984cfaa Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 9 Mar 2021 15:47:29 -0500 Subject: [PATCH 011/104] refactor: rework all end-to-end.rs tests to use the server fixture rather than const connection strings (#948) --- tests/common/server_fixture.rs | 34 ++++++++++++++++++------ tests/end-to-end.rs | 17 +++++------- tests/end_to_end_cases/flight_api.rs | 15 ++++++----- tests/end_to_end_cases/management_cli.rs | 10 ++++--- tests/end_to_end_cases/write_api.rs | 4 +-- tests/end_to_end_cases/write_cli.rs | 6 +++-- 6 files changed, 55 insertions(+), 31 deletions(-) diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 497f45cbcd..6c75c081cf 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -32,9 +32,9 @@ const GRPC_BIND_ADDR: &str = grpc_bind_addr!(); const HTTP_BASE: &str = concat!("http://", http_bind_addr!()); const IOX_API_V1_BASE: &str = concat!("http://", http_bind_addr!(), "/iox/api/v1"); -pub const GRPC_URL_BASE: &str = concat!("http://", grpc_bind_addr!(), "/"); +const GRPC_URL_BASE: &str = concat!("http://", grpc_bind_addr!(), "/"); -pub const TOKEN: &str = "InfluxDB IOx doesn't have authentication yet"; +const TOKEN: &str = "InfluxDB IOx doesn't have authentication yet"; use std::sync::Arc; @@ -45,6 +45,7 @@ use tokio::sync::Mutex; /// testing. pub struct ServerFixture { server: Arc, + grpc_channel: tonic::transport::Channel, } impl ServerFixture { @@ -63,16 +64,27 @@ impl ServerFixture { // ensure the server is ready server.wait_until_ready().await; - ServerFixture { server } + + let grpc_channel = server + .grpc_channel() + .await + .expect("The server should have been up"); + + ServerFixture { + server, + grpc_channel, + } } /// Return a channel connected to the gRPC API. Panics if the /// server is not yet up - pub async fn grpc_channel(&self) -> tonic::transport::Channel { - self.server - .grpc_channel() - .await - .expect("The server should be up") + pub fn grpc_channel(&self) -> tonic::transport::Channel { + self.grpc_channel.clone() + } + + /// Return the url base of the grpc management API + pub fn grpc_url_base(&self) -> &str { + self.server.grpc_url_base() } /// Return the http base URL for the HTTP API @@ -84,6 +96,12 @@ impl ServerFixture { pub fn iox_api_v1_base(&self) -> &str { self.server.iox_api_v1_base() } + + /// Return an a http client suitable suitable for communicating with this + /// server + pub fn influxdb2_client(&self) -> influxdb2_client::Client { + influxdb2_client::Client::new(self.http_base(), TOKEN) + } } #[derive(Debug)] diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index 9d864966f2..ff4e410bac 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -44,13 +44,10 @@ use common::server_fixture::*; async fn read_and_write_data() { let fixture = ServerFixture::create_shared().await; - let influxdb2 = influxdb2_client::Client::new(fixture.http_base(), TOKEN); - let grpc = influxdb_iox_client::connection::Builder::default() - .build(GRPC_URL_BASE) - .await - .unwrap(); - let mut storage_client = StorageClient::new(grpc.clone()); - let mut management_client = influxdb_iox_client::management::Client::new(grpc.clone()); + let influxdb2 = fixture.influxdb2_client(); + let mut storage_client = StorageClient::new(fixture.grpc_channel()); + let mut management_client = + influxdb_iox_client::management::Client::new(fixture.grpc_channel()); // These tests share data; TODO: a better way to indicate this { @@ -65,7 +62,7 @@ async fn read_and_write_data() { read_api::test(&fixture, &scenario, sql_query, &expected_read_data).await; storage_api::test(&mut storage_client, &scenario).await; - flight_api::test(&scenario, sql_query, &expected_read_data).await; + flight_api::test(&fixture, &scenario, sql_query, &expected_read_data).await; } // These tests manage their own data @@ -77,8 +74,8 @@ async fn read_and_write_data() { ) .await; management_api::test(&mut management_client).await; - management_cli::test(GRPC_URL_BASE).await; - write_cli::test(GRPC_URL_BASE).await; + management_cli::test(&fixture).await; + write_cli::test(&fixture).await; test_http_error_messages(&influxdb2).await.unwrap(); } diff --git a/tests/end_to_end_cases/flight_api.rs b/tests/end_to_end_cases/flight_api.rs index 7e8872ac0d..09ecd1ac3d 100644 --- a/tests/end_to_end_cases/flight_api.rs +++ b/tests/end_to_end_cases/flight_api.rs @@ -1,11 +1,14 @@ -use crate::{Scenario, GRPC_URL_BASE}; +use crate::{common::server_fixture::ServerFixture, Scenario}; use arrow_deps::assert_table_eq; -use influxdb_iox_client::{connection::Builder, flight::Client}; +use influxdb_iox_client::flight::Client; -pub async fn test(scenario: &Scenario, sql_query: &str, expected_read_data: &[String]) { - let connection = Builder::default().build(GRPC_URL_BASE).await.unwrap(); - - let mut client = Client::new(connection); +pub async fn test( + server_fixture: &ServerFixture, + scenario: &Scenario, + sql_query: &str, + expected_read_data: &[String], +) { + let mut client = Client::new(server_fixture.grpc_channel()); let mut query_results = client .perform_query(scenario.database_name(), sql_query) diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 56b9b4424a..c2b74d8a5a 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -1,9 +1,13 @@ use assert_cmd::Command; use predicates::prelude::*; -pub async fn test(addr: impl AsRef) { - test_writer_id(addr.as_ref()).await; - test_create_database(addr.as_ref()).await; +use crate::common::server_fixture::ServerFixture; + +pub async fn test(server_fixture: &ServerFixture) { + let addr = server_fixture.grpc_url_base(); + + test_writer_id(addr).await; + test_create_database(addr).await; } async fn test_writer_id(addr: &str) { diff --git a/tests/end_to_end_cases/write_api.rs b/tests/end_to_end_cases/write_api.rs index 89a51c5ac4..276395f4e9 100644 --- a/tests/end_to_end_cases/write_api.rs +++ b/tests/end_to_end_cases/write_api.rs @@ -12,8 +12,8 @@ use crate::common::server_fixture::ServerFixture; async fn test_write() { // TODO sort out changing the test ID let fixture = ServerFixture::create_shared().await; - let mut management_client = management::Client::new(fixture.grpc_channel().await); - let mut write_client = write::Client::new(fixture.grpc_channel().await); + let mut management_client = management::Client::new(fixture.grpc_channel()); + let mut write_client = write::Client::new(fixture.grpc_channel()); const TEST_ID: u32 = 42; diff --git a/tests/end_to_end_cases/write_cli.rs b/tests/end_to_end_cases/write_cli.rs index 575f97da4a..a299d1b0ee 100644 --- a/tests/end_to_end_cases/write_cli.rs +++ b/tests/end_to_end_cases/write_cli.rs @@ -2,11 +2,13 @@ use assert_cmd::Command; use predicates::prelude::*; use test_helpers::make_temp_file; +use crate::common::server_fixture::ServerFixture; + use super::util::rand_name; -pub async fn test(addr: impl AsRef) { +pub async fn test(server_fixture: &ServerFixture) { let db_name = rand_name(); - let addr = addr.as_ref(); + let addr = server_fixture.grpc_url_base(); create_database(&db_name, addr).await; test_write(&db_name, addr).await; } From 28fe808d7e0fbec3b6ea37a690251338a9091e90 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Wed, 10 Mar 2021 11:21:53 +0100 Subject: [PATCH 012/104] fix: Shutdown stest fixture on test harness exit --- scripts/cargo_bin_no_orphan.bash | 31 +++++++++++++++++++++++++ tests/common/mod.rs | 1 + tests/common/no_orphan_cargo.rs | 40 ++++++++++++++++++++++++++++++++ tests/common/server_fixture.rs | 13 ++++------- 4 files changed, 76 insertions(+), 9 deletions(-) create mode 100755 scripts/cargo_bin_no_orphan.bash create mode 100644 tests/common/no_orphan_cargo.rs diff --git a/scripts/cargo_bin_no_orphan.bash b/scripts/cargo_bin_no_orphan.bash new file mode 100755 index 0000000000..ef80ca4e92 --- /dev/null +++ b/scripts/cargo_bin_no_orphan.bash @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Runs a command and waits until it or the parent process dies. +# +# See https://github.com/influxdata/influxdb_iox/issues/951 +# +# Hack self destruction condition: +# when we stop using ServerFixture::create_shared and similar test setup techniques +# we can rely on RAII to properly tear down and remove this hack. +# + +parent=$(ps -o ppid= $$) + +cmd="$1" +shift + +trap 'kill $(jobs -p)' EXIT + +"${cmd}" "$@" & + +child="$!" + +echo child "${child}" >>/tmp/noorphan.log + +check_pid() { + kill -0 "$1" 2>/dev/null +} + +while check_pid "${parent}" && check_pid "${child}"; do + sleep 1 +done diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a057eb4940..1c96171864 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1 +1,2 @@ +mod no_orphan_cargo; pub mod server_fixture; diff --git a/tests/common/no_orphan_cargo.rs b/tests/common/no_orphan_cargo.rs new file mode 100644 index 0000000000..1f0799bd05 --- /dev/null +++ b/tests/common/no_orphan_cargo.rs @@ -0,0 +1,40 @@ +//! Like assert_cmd::cargo::CommandCargoExt +//! but with workaround for https://github.com/influxdata/influxdb_iox/issues/951 + +use std::env; +use std::path; +use std::process::Command; + +const WRAPPER: &str = "scripts/cargo_bin_no_orphan.bash"; + +/// Create a Command to run a specific binary of the current crate. +/// +/// The binary will be executed by a wrapper which ensures it gets +/// killed when the parent process (the caller) dies. +/// This is useful in test fixtures that lazily initialize background processes +/// in static Onces, which are never dropped and thus cannot be cleaned up with +/// RAII. +pub fn cargo_bin>(name: S) -> Command { + let mut cmd = Command::new(WRAPPER); + cmd.arg(cargo_bin_name(name.as_ref())); + cmd +} + +fn cargo_bin_name(name: &str) -> path::PathBuf { + target_dir().join(format!("{}{}", name, env::consts::EXE_SUFFIX)) +} + +// Adapted from +// https://github.com/rust-lang/cargo/blob/485670b3983b52289a2f353d589c57fae2f60f82/tests/testsuite/support/mod.rs#L507 +fn target_dir() -> path::PathBuf { + env::current_exe() + .ok() + .map(|mut path| { + path.pop(); + if path.ends_with("deps") { + path.pop(); + } + path + }) + .unwrap() +} diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 6c75c081cf..0f80b3bb67 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -1,10 +1,7 @@ use std::{fs::File, str}; -use std::{ - num::NonZeroU32, - process::{Child, Command}, -}; +use std::{num::NonZeroU32, process::Child}; -use assert_cmd::prelude::*; +use crate::common::no_orphan_cargo::cargo_bin; use futures::prelude::*; use std::time::Duration; use tempfile::TempDir; @@ -143,8 +140,7 @@ impl TestServer { .expect("cloning file handle for stdout"); let stderr_log_file = log_file; - let server_process = Command::cargo_bin("influxdb_iox") - .unwrap() + let server_process = cargo_bin("influxdb_iox") // Can enable for debbugging //.arg("-vv") .env("INFLUXDB_IOX_ID", "1") @@ -167,8 +163,7 @@ impl TestServer { fn restart(&mut self) -> Result<()> { self.server_process.kill().unwrap(); self.server_process.wait().unwrap(); - self.server_process = Command::cargo_bin("influxdb_iox") - .unwrap() + self.server_process = cargo_bin("influxdb_iox") // Can enable for debbugging //.arg("-vv") .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) From 9ebedf744f9ed846de1098e996b9baf46d5eac78 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 10 Mar 2021 08:21:12 -0500 Subject: [PATCH 013/104] feat: Add CLI to query the database using SQL (#943) * feat: Add CLI to query the database using SQL * fix: merge conflicts --- README.md | 63 +++++---- influxdb_iox_client/src/format.rs | 217 +++++++++++++++++++++++++++++ influxdb_iox_client/src/lib.rs | 3 + src/commands/database.rs | 47 ++++++- src/influxdb_ioxd/http.rs | 84 +++-------- src/influxdb_ioxd/http/format.rs | 127 ----------------- tests/end-to-end.rs | 1 + tests/end_to_end_cases/mod.rs | 1 + tests/end_to_end_cases/read_cli.rs | 166 ++++++++++++++++++++++ 9 files changed, 493 insertions(+), 216 deletions(-) create mode 100644 influxdb_iox_client/src/format.rs delete mode 100644 src/influxdb_ioxd/http/format.rs create mode 100644 tests/end_to_end_cases/read_cli.rs diff --git a/README.md b/README.md index 0dfacaa68b..a576f50e58 100644 --- a/README.md +++ b/README.md @@ -190,34 +190,51 @@ The server will, by default, start an HTTP API server on port `8080` and a gRPC ### Writing and Reading Data Each IOx instance requires a writer ID. -This can be set three ways: +This can be set one of 4 ways: - set an environment variable `INFLUXDB_IOX_ID=42` - set a flag `--writer-id 42` -- send an HTTP PUT request: +- use the API (not convered here) +- use the CLI ```shell -curl --request PUT \ - --url http://localhost:8080/iox/api/v1/id \ - --header 'Content-Type: application/json' \ - --data '{ - "id": 42 - }' +influxdb_iox writer set 42 ``` -To write data, you need a destination database. -This is set via HTTP PUT, identifying the database by org `company` and bucket `sensors`: +To write data, you need to create a database. You can do so via the API or using the CLI. For example, to create a database called `company_sensors` with a 100MB mutable buffer, use this command: + ```shell -curl --request PUT \ - --url http://localhost:8080/iox/api/v1/databases/company_sensors \ - --header 'Content-Type: application/json' \ - --data '{ -}' +influxdb_iox database create company_sensors -m 100 ``` -Data can be stored in InfluxDB IOx by sending it in [line protocol] format to the `/api/v2/write` -endpoint. Data is stored by organization and bucket names. Here's an example using [`curl`] with -the organization name `company` and the bucket name `sensors` that will send the data in the -`tests/fixtures/lineproto/metrics.lp` file in this repository, assuming that you're running the -server on the default port: +Data can be stored in InfluxDB IOx by sending it in [line protocol] +format to the `/api/v2/write` endpoint or using the CLI. For example, +here is a command that will send the data in the +`tests/fixtures/lineproto/metrics.lp` file in this repository, +assuming that you're running the server on the default port into +the `company_sensors` database, you can use: + +```shell +influxdb_iox database write company_sensors tests/fixtures/lineproto/metrics.lp +``` + +To query data stored in the `company_sensors` database: + +```shell +influxdb_iox database query company_sensors "SELECT * FROM cpu LIMIT 10" +``` + + +### InfluxDB 2.0 compatibility + +InfluxDB IOx allows seamless interoperability with InfluxDB 2.0. + +InfluxDB 2.0 stores data in organization and buckets, but InfluxDB IOx +stores data in named databases. IOx maps `organization` and `bucket` +to a database named with the two parts separated by an underscore +(`_`): `organization_bucket`. + +Here's an example using [`curl`] command to send the same data into +the `company_sensors` database using the InfluxDB 2.0 `/api/v2/write` +API: ```shell curl -v "http://127.0.0.1:8080/api/v2/write?org=company&bucket=sensors" --data-binary @tests/fixtures/lineproto/metrics.lp @@ -226,12 +243,6 @@ curl -v "http://127.0.0.1:8080/api/v2/write?org=company&bucket=sensors" --data-b [line protocol]: https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/ [`curl`]: https://curl.se/ -To query stored data, use the `/api/v2/read` endpoint with a SQL query. This example will return -all data in the `company` organization's `sensors` bucket for the `processes` measurement: - -```shell -curl -v -G -d 'org=company' -d 'bucket=sensors' --data-urlencode 'sql_query=select * from processes' "http://127.0.0.1:8080/api/v2/read" -``` ### Health Checks diff --git a/influxdb_iox_client/src/format.rs b/influxdb_iox_client/src/format.rs new file mode 100644 index 0000000000..4e05efe7fa --- /dev/null +++ b/influxdb_iox_client/src/format.rs @@ -0,0 +1,217 @@ +//! Output formatting utilities for Arrow record batches + +use std::{fmt::Display, str::FromStr}; + +use thiserror::Error; + +use arrow_deps::arrow::{ + self, csv::WriterBuilder, error::ArrowError, json::ArrayWriter, record_batch::RecordBatch, +}; + +/// Error type for results formatting +#[derive(Debug, Error)] +pub enum Error { + /// Unknown formatting type + #[error("Unknown format type: {}. Expected one of 'pretty', 'csv' or 'json'", .0)] + Invalid(String), + + /// Error pretty printing + #[error("Arrow pretty printing error: {}", .0)] + PrettyArrow(ArrowError), + + /// Error during CSV conversion + #[error("Arrow csv printing error: {}", .0)] + CsvArrow(ArrowError), + + /// Error during JSON conversion + #[error("Arrow json printing error: {}", .0)] + JsonArrow(ArrowError), + + /// Error converting CSV output to utf-8 + #[error("Error converting CSV output to UTF-8: {}", .0)] + CsvUtf8(std::string::FromUtf8Error), + + /// Error converting JSON output to utf-8 + #[error("Error converting JSON output to UTF-8: {}", .0)] + JsonUtf8(std::string::FromUtf8Error), +} +type Result = std::result::Result; + +#[derive(Debug, Copy, Clone, PartialEq)] +/// Requested output format for the query endpoint +pub enum QueryOutputFormat { + /// Arrow pretty printer format (default) + Pretty, + /// Comma separated values + CSV, + /// Arrow JSON format + JSON, +} + +impl Display for QueryOutputFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + QueryOutputFormat::Pretty => write!(f, "pretty"), + QueryOutputFormat::CSV => write!(f, "csv"), + QueryOutputFormat::JSON => write!(f, "json"), + } + } +} + +impl Default for QueryOutputFormat { + fn default() -> Self { + Self::Pretty + } +} + +impl FromStr for QueryOutputFormat { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "pretty" => Ok(Self::Pretty), + "csv" => Ok(Self::CSV), + "json" => Ok(Self::JSON), + _ => Err(Error::Invalid(s.to_string())), + } + } +} + +impl QueryOutputFormat { + /// Return the Mcontent-type of this format + pub fn content_type(&self) -> &'static str { + match self { + Self::Pretty => "text/plain", + Self::CSV => "text/csv", + Self::JSON => "application/json", + } + } +} + +impl QueryOutputFormat { + /// Format the [`RecordBatch`]es into a String in one of the + /// following formats: + /// + /// Pretty: + /// ```text + /// +----------------+--------------+-------+-----------------+------------+ + /// | bottom_degrees | location | state | surface_degrees | time | + /// +----------------+--------------+-------+-----------------+------------+ + /// | 50.4 | santa_monica | CA | 65.2 | 1568756160 | + /// +----------------+--------------+-------+-----------------+------------+ + /// ``` + /// + /// CSV: + /// ```text + /// bottom_degrees,location,state,surface_degrees,time + /// 50.4,santa_monica,CA,65.2,1568756160 + /// ``` + /// + /// JSON: + /// + /// Example (newline + whitespace added for clarity): + /// ```text + /// [ + /// {"bottom_degrees":50.4,"location":"santa_monica","state":"CA","surface_degrees":65.2,"time":1568756160}, + /// {"location":"Boston","state":"MA","surface_degrees":50.2,"time":1568756160} + /// ] + /// ``` + pub fn format(&self, batches: &[RecordBatch]) -> Result { + match self { + Self::Pretty => batches_to_pretty(&batches), + Self::CSV => batches_to_csv(&batches), + Self::JSON => batches_to_json(&batches), + } + } +} + +fn batches_to_pretty(batches: &[RecordBatch]) -> Result { + arrow::util::pretty::pretty_format_batches(batches).map_err(Error::PrettyArrow) +} + +fn batches_to_csv(batches: &[RecordBatch]) -> Result { + let mut bytes = vec![]; + + { + let mut writer = WriterBuilder::new().has_headers(true).build(&mut bytes); + + for batch in batches { + writer.write(batch).map_err(Error::CsvArrow)?; + } + } + let csv = String::from_utf8(bytes).map_err(Error::CsvUtf8)?; + Ok(csv) +} + +fn batches_to_json(batches: &[RecordBatch]) -> Result { + let mut bytes = vec![]; + + { + let mut writer = ArrayWriter::new(&mut bytes); + writer.write_batches(batches).map_err(Error::CsvArrow)?; + + writer.finish().map_err(Error::CsvArrow)?; + } + + let json = String::from_utf8(bytes).map_err(Error::JsonUtf8)?; + + Ok(json) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_from_str() { + assert_eq!( + QueryOutputFormat::from_str("pretty").unwrap(), + QueryOutputFormat::Pretty + ); + assert_eq!( + QueryOutputFormat::from_str("pRetty").unwrap(), + QueryOutputFormat::Pretty + ); + + assert_eq!( + QueryOutputFormat::from_str("csv").unwrap(), + QueryOutputFormat::CSV + ); + assert_eq!( + QueryOutputFormat::from_str("CSV").unwrap(), + QueryOutputFormat::CSV + ); + + assert_eq!( + QueryOutputFormat::from_str("json").unwrap(), + QueryOutputFormat::JSON + ); + assert_eq!( + QueryOutputFormat::from_str("JSON").unwrap(), + QueryOutputFormat::JSON + ); + + assert_eq!( + QueryOutputFormat::from_str("un").unwrap_err().to_string(), + "Unknown format type: un. Expected one of 'pretty', 'csv' or 'json'" + ); + } + + #[test] + fn test_from_roundtrip() { + assert_eq!( + QueryOutputFormat::from_str(&QueryOutputFormat::Pretty.to_string()).unwrap(), + QueryOutputFormat::Pretty + ); + + assert_eq!( + QueryOutputFormat::from_str(&QueryOutputFormat::CSV.to_string()).unwrap(), + QueryOutputFormat::CSV + ); + + assert_eq!( + QueryOutputFormat::from_str(&QueryOutputFormat::JSON.to_string()).unwrap(), + QueryOutputFormat::JSON + ); + } +} diff --git a/influxdb_iox_client/src/lib.rs b/influxdb_iox_client/src/lib.rs index ddb713535b..855e41a39c 100644 --- a/influxdb_iox_client/src/lib.rs +++ b/influxdb_iox_client/src/lib.rs @@ -16,4 +16,7 @@ pub use client::flight; /// Builder for constructing connections for use with the various gRPC clients pub mod connection; +/// Output formatting utilities +pub mod format; + mod client; diff --git a/src/commands/database.rs b/src/commands/database.rs index 3ec3a6e149..5efbc80847 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -1,8 +1,10 @@ //! This module implements the `database` CLI command -use std::{fs::File, io::Read, path::PathBuf}; +use std::{fs::File, io::Read, path::PathBuf, str::FromStr}; use influxdb_iox_client::{ connection::Builder, + flight, + format::QueryOutputFormat, management::{ self, generated_types::*, CreateDatabaseError, GetDatabaseError, ListDatabaseError, }, @@ -33,6 +35,12 @@ pub enum Error { #[error("Error writing: {0}")] WriteError(#[from] WriteError), + + #[error("Error formatting: {0}")] + FormattingError(#[from] influxdb_iox_client::format::Error), + + #[error("Error querying: {0}")] + Query(#[from] influxdb_iox_client::flight::Error), } pub type Result = std::result::Result; @@ -72,12 +80,27 @@ struct Write { file_name: PathBuf, } +/// Query the data with SQL +#[derive(Debug, StructOpt)] +struct Query { + /// The name of the database + name: String, + + /// The query to run, in SQL format + query: String, + + /// Optional format ('pretty', 'json', or 'csv') + #[structopt(short, long, default_value = "pretty")] + format: String, +} + /// All possible subcommands for database #[derive(Debug, StructOpt)] enum Command { Create(Create), Get(Get), Write(Write), + Query(Query), } pub async fn command(url: String, config: Config) -> Result<()> { @@ -130,6 +153,28 @@ pub async fn command(url: String, config: Config) -> Result<()> { println!("{} Lines OK", lines_written); } + Command::Query(query) => { + let mut client = flight::Client::new(connection); + let Query { + name, + format, + query, + } = query; + + let format = QueryOutputFormat::from_str(&format)?; + + let mut query_results = client.perform_query(&name, query).await?; + + // It might be nice to do some sort of streaming write + // rather than buffering the whole thing. + let mut batches = vec![]; + while let Some(data) = query_results.next().await? { + batches.push(data); + } + + let formatted_result = format.format(&batches)?; + println!("{}", formatted_result); + } } Ok(()) diff --git a/src/influxdb_ioxd/http.rs b/src/influxdb_ioxd/http.rs index 8eaf090b24..8e057f98e4 100644 --- a/src/influxdb_ioxd/http.rs +++ b/src/influxdb_ioxd/http.rs @@ -18,6 +18,7 @@ use data_types::{ names::{org_and_bucket_to_database, OrgBucketMappingError}, DatabaseName, }; +use influxdb_iox_client::format::QueryOutputFormat; use influxdb_line_protocol::parse_lines; use object_store::ObjectStoreApi; use query::{frontend::sql::SQLQueryPlanner, Database, DatabaseStore}; @@ -34,10 +35,11 @@ use snafu::{OptionExt, ResultExt, Snafu}; use tracing::{debug, error, info}; use data_types::http::WalMetadataResponse; -use std::{fmt::Debug, str, sync::Arc}; - -mod format; -use format::QueryOutputFormat; +use std::{ + fmt::Debug, + str::{self, FromStr}, + sync::Arc, +}; /// Constants used in API error codes. /// @@ -187,6 +189,12 @@ pub enum ApplicationError { #[snafu(display("Internal error creating HTTP response: {}", source))] CreatingResponse { source: http::Error }, + #[snafu(display("Invalid format '{}': : {}", format, source))] + ParsingFormat { + format: String, + source: influxdb_iox_client::format::Error, + }, + #[snafu(display( "Error formatting results of SQL query '{}' using '{:?}': {}", q, @@ -196,7 +204,7 @@ pub enum ApplicationError { FormattingResult { q: String, format: QueryOutputFormat, - source: format::Error, + source: influxdb_iox_client::format::Error, }, } @@ -230,6 +238,7 @@ impl ApplicationError { Self::WALNotFound { .. } => self.not_found(), Self::CreatingResponse { .. } => self.internal_error(), Self::FormattingResult { .. } => self.internal_error(), + Self::ParsingFormat { .. } => self.bad_request(), } } @@ -458,8 +467,12 @@ where /// Parsed URI Parameters of the request to the .../query endpoint struct QueryParams { q: String, - #[serde(default)] - format: QueryOutputFormat, + #[serde(default = "default_format")] + format: String, +} + +fn default_format() -> String { + QueryOutputFormat::default().to_string() } #[tracing::instrument(level = "debug")] @@ -475,6 +488,8 @@ async fn query( query_string: uri_query, })?; + let format = QueryOutputFormat::from_str(&format).context(ParsingFormat { format })?; + let db_name_str = req .param("name") .expect("db name must have been set by routerify") @@ -1420,59 +1435,4 @@ mod tests { collect(physical_plan).await.unwrap() } - - #[test] - fn query_params_format_default() { - // default to pretty format when not otherwise specified - assert_eq!( - serde_urlencoded::from_str("q=foo"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::Pretty - }) - ); - } - - #[test] - fn query_params_format_pretty() { - assert_eq!( - serde_urlencoded::from_str("q=foo&format=pretty"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::Pretty - }) - ); - } - - #[test] - fn query_params_format_csv() { - assert_eq!( - serde_urlencoded::from_str("q=foo&format=csv"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::CSV - }) - ); - } - - #[test] - fn query_params_format_json() { - assert_eq!( - serde_urlencoded::from_str("q=foo&format=json"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::JSON - }) - ); - } - - #[test] - fn query_params_bad_format() { - assert_eq!( - serde_urlencoded::from_str::("q=foo&format=jsob") - .unwrap_err() - .to_string(), - "unknown variant `jsob`, expected one of `pretty`, `csv`, `json`" - ); - } } diff --git a/src/influxdb_ioxd/http/format.rs b/src/influxdb_ioxd/http/format.rs deleted file mode 100644 index 1f13493aae..0000000000 --- a/src/influxdb_ioxd/http/format.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! Output formatting utilities for query endpoint - -use serde::Deserialize; -use snafu::{ResultExt, Snafu}; - -use arrow_deps::arrow::{ - self, csv::WriterBuilder, error::ArrowError, json::ArrayWriter, record_batch::RecordBatch, -}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("Arrow pretty printing error: {}", source))] - PrettyArrow { source: ArrowError }, - - #[snafu(display("Arrow csv printing error: {}", source))] - CsvArrow { source: ArrowError }, - - #[snafu(display("Arrow json printing error: {}", source))] - JsonArrow { source: ArrowError }, - - #[snafu(display("Error converting CSV output to UTF-8: {}", source))] - CsvUtf8 { source: std::string::FromUtf8Error }, - - #[snafu(display("Error converting JSON output to UTF-8: {}", source))] - JsonUtf8 { source: std::string::FromUtf8Error }, -} -type Result = std::result::Result; - -#[derive(Deserialize, Debug, Copy, Clone, PartialEq)] -/// Requested output format for the query endpoint -pub enum QueryOutputFormat { - /// Arrow pretty printer format (default) - #[serde(rename = "pretty")] - Pretty, - /// Comma separated values - #[serde(rename = "csv")] - CSV, - /// Arrow JSON format - #[serde(rename = "json")] - JSON, -} - -impl Default for QueryOutputFormat { - fn default() -> Self { - Self::Pretty - } -} - -impl QueryOutputFormat { - /// Return the content type of the relevant format - pub fn content_type(&self) -> &'static str { - match self { - Self::Pretty => "text/plain", - Self::CSV => "text/csv", - Self::JSON => "application/json", - } - } -} - -impl QueryOutputFormat { - /// Format the [`RecordBatch`]es into a String in one of the - /// following formats: - /// - /// Pretty: - /// ```text - /// +----------------+--------------+-------+-----------------+------------+ - /// | bottom_degrees | location | state | surface_degrees | time | - /// +----------------+--------------+-------+-----------------+------------+ - /// | 50.4 | santa_monica | CA | 65.2 | 1568756160 | - /// +----------------+--------------+-------+-----------------+------------+ - /// ``` - /// - /// CSV: - /// ```text - /// bottom_degrees,location,state,surface_degrees,time - /// 50.4,santa_monica,CA,65.2,1568756160 - /// ``` - /// - /// JSON: - /// - /// Example (newline + whitespace added for clarity): - /// ```text - /// [ - /// {"bottom_degrees":50.4,"location":"santa_monica","state":"CA","surface_degrees":65.2,"time":1568756160}, - /// {"location":"Boston","state":"MA","surface_degrees":50.2,"time":1568756160} - /// ] - /// ``` - pub fn format(&self, batches: &[RecordBatch]) -> Result { - match self { - Self::Pretty => batches_to_pretty(&batches), - Self::CSV => batches_to_csv(&batches), - Self::JSON => batches_to_json(&batches), - } - } -} - -fn batches_to_pretty(batches: &[RecordBatch]) -> Result { - arrow::util::pretty::pretty_format_batches(batches).context(PrettyArrow) -} - -fn batches_to_csv(batches: &[RecordBatch]) -> Result { - let mut bytes = vec![]; - - { - let mut writer = WriterBuilder::new().has_headers(true).build(&mut bytes); - - for batch in batches { - writer.write(batch).context(CsvArrow)?; - } - } - let csv = String::from_utf8(bytes).context(CsvUtf8)?; - Ok(csv) -} - -fn batches_to_json(batches: &[RecordBatch]) -> Result { - let mut bytes = vec![]; - - { - let mut writer = ArrayWriter::new(&mut bytes); - writer.write_batches(batches).context(CsvArrow)?; - writer.finish().context(CsvArrow)?; - } - - let json = String::from_utf8(bytes).context(JsonUtf8)?; - - Ok(json) -} diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index ff4e410bac..46c87a2535 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -76,6 +76,7 @@ async fn read_and_write_data() { management_api::test(&mut management_client).await; management_cli::test(&fixture).await; write_cli::test(&fixture).await; + read_cli::test(&fixture).await; test_http_error_messages(&influxdb2).await.unwrap(); } diff --git a/tests/end_to_end_cases/mod.rs b/tests/end_to_end_cases/mod.rs index 8584f185f4..ab69c33893 100644 --- a/tests/end_to_end_cases/mod.rs +++ b/tests/end_to_end_cases/mod.rs @@ -2,6 +2,7 @@ pub mod flight_api; pub mod management_api; pub mod management_cli; pub mod read_api; +pub mod read_cli; pub mod storage_api; pub mod util; pub mod write_api; diff --git a/tests/end_to_end_cases/read_cli.rs b/tests/end_to_end_cases/read_cli.rs new file mode 100644 index 0000000000..5a5e8673ad --- /dev/null +++ b/tests/end_to_end_cases/read_cli.rs @@ -0,0 +1,166 @@ +use assert_cmd::Command; +use predicates::prelude::*; +use test_helpers::make_temp_file; + +use crate::common::server_fixture::ServerFixture; + +use super::util::rand_name; + +pub async fn test(server_fixture: &ServerFixture) { + let db_name = rand_name(); + let addr = server_fixture.grpc_url_base(); + create_database(&db_name, addr).await; + test_read_default(&db_name, addr).await; + test_read_format_pretty(&db_name, addr).await; + test_read_format_csv(&db_name, addr).await; + test_read_format_json(&db_name, addr).await; + test_read_error(&db_name, addr).await; +} + +async fn create_database(db_name: &str, addr: &str) { + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db_name) + .arg("-m") + .arg("100") // give it a mutable buffer + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + ]; + + let lp_data_file = make_temp_file(lp_data.join("\n")); + + // read from temp file + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(db_name) + .arg(lp_data_file.as_ref()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("2 Lines OK")); +} + +async fn test_read_default(db_name: &str, addr: &str) { + let expected = "+--------+------+------+\n\ + | region | time | user |\n\ + +--------+------+------+\n\ + | west | 100 | 23.2 |\n\ + | west | 150 | 21 |\n\ + +--------+------+------+"; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_format_pretty(db_name: &str, addr: &str) { + let expected = "+--------+------+------+\n\ + | region | time | user |\n\ + +--------+------+------+\n\ + | west | 100 | 23.2 |\n\ + | west | 150 | 21 |\n\ + +--------+------+------+"; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("pretty") + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_format_csv(db_name: &str, addr: &str) { + let expected = "region,time,user\nwest,100,23.2\nwest,150,21.0"; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("csv") + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_format_json(db_name: &str, addr: &str) { + let expected = + r#"[{"region":"west","time":100,"user":23.2},{"region":"west","time":150,"user":21.0}]"#; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("json") + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_error(db_name: &str, addr: &str) { + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from unknown_table") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains( + "no chunks found in builder for table", + )); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("not_a_valid_format") + .assert() + .failure() + .stderr(predicate::str::contains( + "Unknown format type: not_a_valid_format. Expected one of 'pretty', 'csv' or 'json'", + )); +} From 1b40ca7ab5e2ecbacd6a41afbed88af6c3b95392 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 8 Mar 2021 19:32:41 +0100 Subject: [PATCH 014/104] feat: Implement CRUD for remote addresses --- .../iox/management/v1/service.proto | 42 ++++++++++++++ influxdb_iox_client/src/client/management.rs | 53 +++++++++++++++++ server/src/config.rs | 25 +++++++- server/src/lib.rs | 17 +++++- src/influxdb_ioxd/rpc/management.rs | 47 +++++++++++++++ tests/end_to_end_cases/management_api.rs | 57 +++++++++++++++++++ 6 files changed, 239 insertions(+), 2 deletions(-) diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index 73735431c1..f8fb6edd1e 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -14,6 +14,15 @@ service ManagementService { rpc GetDatabase(GetDatabaseRequest) returns (GetDatabaseResponse); rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + + // List remote IOx servers we know about. + rpc ListRemotes(ListRemotesRequest) returns (ListRemotesResponse); + + // Update information about a remote IOx server (upsert). + rpc UpdateRemote(UpdateRemoteRequest) returns (UpdateRemoteResponse); + + // Delete a reference to remote IOx server. + rpc DeleteRemote(DeleteRemoteRequest) returns (DeleteRemoteResponse); } message GetWriterIdRequest {} @@ -47,3 +56,36 @@ message CreateDatabaseRequest { } message CreateDatabaseResponse {} + +message ListRemotesRequest {} + +message ListRemotesResponse { + repeated Remote remotes = 1; +} + +// This resource represents a remote IOx server. +message Remote { + // The writer ID associated with a remote IOx server. + uint32 id = 1; + + // The address of the remote IOx server gRPC endpoint. + string connection_string = 2; +} + +// Updates information about a remote IOx server. +// +// If a remote for a given `id` already exists, it is updated in place. +message UpdateRemoteRequest { + // If omitted, the remote associated with `id` will be removed. + Remote remote = 1; + + // TODO(#917): add an optional flag to test the connection or not before adding it. +} + +message UpdateRemoteResponse {} + +message DeleteRemoteRequest{ + uint32 id = 1; +} + +message DeleteRemoteResponse {} diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index 2b7c2b1fee..df98aaf891 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -80,6 +80,22 @@ pub enum GetDatabaseError { ServerError(tonic::Status), } +/// Errors returned by Client::list_remotes +#[derive(Debug, Error)] +pub enum ListRemotesError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::update_remote +#[derive(Debug, Error)] +pub enum UpdateRemoteError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// An IOx Management API client. /// /// ```no_run @@ -198,4 +214,41 @@ impl Client { .ok_or(GetDatabaseError::EmptyResponse)?; Ok(rules) } + + /// List remotes. + pub async fn list_remotes(&mut self) -> Result, ListRemotesError> { + let response = self + .inner + .list_remotes(ListRemotesRequest {}) + .await + .map_err(ListRemotesError::ServerError)?; + Ok(response.into_inner().remotes) + } + + /// Update remote + pub async fn update_remote( + &mut self, + id: u32, + connection_string: impl Into, + ) -> Result<(), UpdateRemoteError> { + self.inner + .update_remote(UpdateRemoteRequest { + remote: Some(generated_types::Remote { + id, + connection_string: connection_string.into(), + }), + }) + .await + .map_err(UpdateRemoteError::ServerError)?; + Ok(()) + } + + /// Delete remote + pub async fn delete_remote(&mut self, id: u32) -> Result<(), UpdateRemoteError> { + self.inner + .delete_remote(DeleteRemoteRequest { id }) + .await + .map_err(UpdateRemoteError::ServerError)?; + Ok(()) + } } diff --git a/server/src/config.rs b/server/src/config.rs index 7c702af1f4..75d8f8b1ed 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -1,6 +1,9 @@ /// This module contains code for managing the configuration of the server. use crate::{db::Db, Error, Result}; -use data_types::{database_rules::DatabaseRules, DatabaseName}; +use data_types::{ + database_rules::{DatabaseRules, WriterId}, + DatabaseName, +}; use mutable_buffer::MutableBufferDb; use object_store::path::ObjectStorePath; use read_buffer::Database as ReadBufferDb; @@ -62,6 +65,21 @@ impl Config { state.databases.keys().cloned().collect() } + pub(crate) fn remotes_sorted(&self) -> Vec<(WriterId, String)> { + let state = self.state.read().expect("mutex poisoned"); + state.remotes.iter().map(|(&a, b)| (a, b.clone())).collect() + } + + pub(crate) fn update_remote(&self, id: WriterId, addr: GRPCConnectionString) { + let mut state = self.state.write().expect("mutex poisoned"); + state.remotes.insert(id, addr); + } + + pub(crate) fn delete_remote(&self, id: WriterId) -> Option { + let mut state = self.state.write().expect("mutex poisoned"); + state.remotes.remove(&id) + } + fn commit(&self, name: &DatabaseName<'static>, db: Arc) { let mut state = self.state.write().expect("mutex poisoned"); let name = state @@ -87,10 +105,15 @@ pub fn object_store_path_for_database_config( path } +/// A gRPC connection string. +pub type GRPCConnectionString = String; + #[derive(Default, Debug)] struct ConfigState { reservations: BTreeSet>, databases: BTreeMap, Arc>, + /// Map between remote IOx server IDs and management API connection strings. + remotes: BTreeMap, } /// CreateDatabaseHandle is retunred when a call is made to `create_db` on diff --git a/server/src/lib.rs b/server/src/lib.rs index 06fab7c3a0..e271fdd960 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -83,7 +83,9 @@ use std::sync::{ use crate::{ buffer::SegmentPersistenceTask, - config::{object_store_path_for_database_config, Config, DB_RULES_FILE_NAME}, + config::{ + object_store_path_for_database_config, Config, GRPCConnectionString, DB_RULES_FILE_NAME, + }, db::Db, tracker::TrackerRegistry, }; @@ -98,6 +100,7 @@ use query::{exec::Executor, DatabaseStore}; use async_trait::async_trait; use bytes::Bytes; +use data_types::database_rules::WriterId; use futures::stream::TryStreamExt; use snafu::{OptionExt, ResultExt, Snafu}; use tracing::error; @@ -367,6 +370,18 @@ impl Server { pub async fn db_rules(&self, name: &DatabaseName<'_>) -> Option { self.config.db(name).map(|d| d.rules.clone()) } + + pub fn remotes_sorted(&self) -> Vec<(WriterId, String)> { + self.config.remotes_sorted() + } + + pub fn update_remote(&self, id: WriterId, addr: GRPCConnectionString) { + self.config.update_remote(id, addr) + } + + pub fn delete_remote(&self, id: WriterId) -> Option { + self.config.delete_remote(id) + } } #[async_trait] diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index 102b8b7c2b..950dcbe941 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -95,6 +95,53 @@ where Err(e) => Err(default_error_handler(e)), } } + + async fn list_remotes( + &self, + _: Request, + ) -> Result, Status> { + let remotes = self + .server + .remotes_sorted() + .into_iter() + .map(|(id, connection_string)| Remote { + id, + connection_string, + }) + .collect(); + Ok(Response::new(ListRemotesResponse { remotes })) + } + + async fn update_remote( + &self, + request: Request, + ) -> Result, Status> { + let remote = request + .into_inner() + .remote + .ok_or_else(|| FieldViolation::required("remote"))?; + if remote.id == 0 { + return Err(FieldViolation::required("id").scope("remote").into()); + } + self.server + .update_remote(remote.id, remote.connection_string); + Ok(Response::new(UpdateRemoteResponse {})) + } + + async fn delete_remote( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.id == 0 { + return Err(FieldViolation::required("id").into()); + } + self.server + .delete_remote(request.id) + .ok_or_else(NotFound::default)?; + + Ok(Response::new(DeleteRemoteResponse {})) + } } pub fn make_server( diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 802f0a36e8..ac3df4b1a4 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -7,6 +7,7 @@ use influxdb_iox_client::management::{Client, CreateDatabaseError}; use super::util::rand_name; pub async fn test(client: &mut Client) { + test_list_update_remotes(client).await; test_set_get_writer_id(client).await; test_create_database_duplicate_name(client).await; test_create_database_invalid_name(client).await; @@ -14,6 +15,62 @@ pub async fn test(client: &mut Client) { test_create_get_database(client).await; } +async fn test_list_update_remotes(client: &mut Client) { + const TEST_REMOTE_ID_1: u32 = 42; + const TEST_REMOTE_ADDR_1: &str = "1.2.3.4:1234"; + const TEST_REMOTE_ID_2: u32 = 84; + const TEST_REMOTE_ADDR_2: &str = "4.3.2.1:4321"; + const TEST_REMOTE_ADDR_2_UPDATED: &str = "40.30.20.10:4321"; + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 0); + + client + .update_remote(TEST_REMOTE_ID_1, TEST_REMOTE_ADDR_1) + .await + .expect("update failed"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 1); + + client + .update_remote(TEST_REMOTE_ID_2, TEST_REMOTE_ADDR_2) + .await + .expect("update failed"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 2); + assert_eq!(res[0].id, TEST_REMOTE_ID_1); + assert_eq!(res[0].connection_string, TEST_REMOTE_ADDR_1); + assert_eq!(res[1].id, TEST_REMOTE_ID_2); + assert_eq!(res[1].connection_string, TEST_REMOTE_ADDR_2); + + client + .delete_remote(TEST_REMOTE_ID_1) + .await + .expect("delete failed"); + + client + .delete_remote(TEST_REMOTE_ID_1) + .await + .expect_err("expected delete to fail"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 1); + assert_eq!(res[0].id, TEST_REMOTE_ID_2); + assert_eq!(res[0].connection_string, TEST_REMOTE_ADDR_2); + + client + .update_remote(TEST_REMOTE_ID_2, TEST_REMOTE_ADDR_2_UPDATED) + .await + .expect("update failed"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 1); + assert_eq!(res[0].id, TEST_REMOTE_ID_2); + assert_eq!(res[0].connection_string, TEST_REMOTE_ADDR_2_UPDATED); +} + async fn test_set_get_writer_id(client: &mut Client) { const TEST_ID: u32 = 42; From 1c6d25e7d88c52a4d2fd4186f8d7a585a0a76234 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 10 Mar 2021 09:05:35 -0500 Subject: [PATCH 015/104] refactor: Port end-to-end tests that modify global state to their own fixture (#949) * refactor: Use autogenerated ports in end-to-end.rs * refactor: Use different fixture for tests that change writer id * fix: clippy --- tests/common/server_fixture.rs | 157 ++++++++++++++++------- tests/end-to-end.rs | 32 ++--- tests/end_to_end_cases/management_api.rs | 20 ++- tests/end_to_end_cases/management_cli.rs | 6 +- tests/end_to_end_cases/read_cli.rs | 7 +- tests/end_to_end_cases/write_api.rs | 10 -- tests/end_to_end_cases/write_cli.rs | 6 +- 7 files changed, 143 insertions(+), 95 deletions(-) diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 0f80b3bb67..a40644bc02 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -1,4 +1,8 @@ -use std::{fs::File, str}; +use std::{ + fs::File, + str, + sync::atomic::{AtomicUsize, Ordering::SeqCst}, +}; use std::{num::NonZeroU32, process::Child}; use crate::common::no_orphan_cargo::cargo_bin; @@ -13,23 +17,45 @@ type Result = std::result::Result; // running locally. // TODO(786): allocate random free ports instead of hardcoding. // TODO(785): we cannot use localhost here. -macro_rules! http_bind_addr { - () => { - "127.0.0.1:8090" - }; -} -macro_rules! grpc_bind_addr { - () => { - "127.0.0.1:8092" - }; +static NEXT_PORT: AtomicUsize = AtomicUsize::new(8090); + +/// This structure contains all the addresses a test server should use +struct BindAddresses { + http_port: usize, + grpc_port: usize, + + http_bind_addr: String, + grpc_bind_addr: String, + + http_base: String, + iox_api_v1_base: String, + grpc_base: String, } -const HTTP_BIND_ADDR: &str = http_bind_addr!(); -const GRPC_BIND_ADDR: &str = grpc_bind_addr!(); +impl BindAddresses { + /// return a new port assignment suitable for this test's use + fn new() -> Self { + let http_port = NEXT_PORT.fetch_add(1, SeqCst); + let grpc_port = NEXT_PORT.fetch_add(1, SeqCst); -const HTTP_BASE: &str = concat!("http://", http_bind_addr!()); -const IOX_API_V1_BASE: &str = concat!("http://", http_bind_addr!(), "/iox/api/v1"); -const GRPC_URL_BASE: &str = concat!("http://", grpc_bind_addr!(), "/"); + let http_bind_addr = format!("127.0.0.1:{}", http_port); + let grpc_bind_addr = format!("127.0.0.1:{}", grpc_port); + + let http_base = format!("http://{}", http_bind_addr); + let iox_api_v1_base = format!("http://{}/iox/api/v1", http_bind_addr); + let grpc_base = format!("http://{}", grpc_bind_addr); + + Self { + http_port, + grpc_port, + http_bind_addr, + grpc_bind_addr, + http_base, + iox_api_v1_base, + grpc_base, + } + } +} const TOKEN: &str = "InfluxDB IOx doesn't have authentication yet"; @@ -45,9 +71,20 @@ pub struct ServerFixture { grpc_channel: tonic::transport::Channel, } +/// Specifieds should we configure a server initially +enum InitialConfig { + /// Set the writer id to something so it can accept writes + SetWriterId, + + /// leave the writer id empty so the test can set it + None, +} + impl ServerFixture { /// Create a new server fixture and wait for it to be ready. This - /// is called "create" rather than new because it is async and waits + /// is called "create" rather than new because it is async and + /// waits. The shared database is configured with a writer id and + /// can be used immediately /// /// This is currently implemented as a singleton so all tests *must* /// use a new database and not interfere with the existing database. @@ -60,8 +97,24 @@ impl ServerFixture { })); // ensure the server is ready - server.wait_until_ready().await; + server.wait_until_ready(InitialConfig::SetWriterId).await; + Self::create_common(server).await + } + /// Create a new server fixture and wait for it to be ready. This + /// is called "create" rather than new because it is async and + /// waits. The database is left unconfigured (no writer id) and + /// is not shared with any other tests. + pub async fn create_single_use() -> Self { + let server = TestServer::new().expect("Could start test server"); + let server = Arc::new(server); + + // ensure the server is ready + server.wait_until_ready(InitialConfig::None).await; + Self::create_common(server).await + } + + async fn create_common(server: Arc) -> Self { let grpc_channel = server .grpc_channel() .await @@ -80,18 +133,18 @@ impl ServerFixture { } /// Return the url base of the grpc management API - pub fn grpc_url_base(&self) -> &str { - self.server.grpc_url_base() + pub fn grpc_base(&self) -> &str { + &self.server.addrs().grpc_base } /// Return the http base URL for the HTTP API pub fn http_base(&self) -> &str { - self.server.http_base() + &self.server.addrs().http_base } /// Return the base URL for the IOx V1 API pub fn iox_api_v1_base(&self) -> &str { - self.server.iox_api_v1_base() + &self.server.addrs().iox_api_v1_base } /// Return an a http client suitable suitable for communicating with this @@ -113,8 +166,12 @@ struct TestServer { /// Is the server ready to accept connections? ready: Mutex, + /// Handle to the server process being controlled server_process: Child, + /// Which ports this server should use + addrs: BindAddresses, + // The temporary directory **must** be last so that it is // dropped after the database closes. dir: TempDir, @@ -122,13 +179,17 @@ struct TestServer { impl TestServer { fn new() -> Result { + let addrs = BindAddresses::new(); let ready = Mutex::new(ServerState::Started); let dir = test_helpers::tmp_dir().unwrap(); // Make a log file in the temporary dir (don't auto delete it to help debugging // efforts) let mut log_path = std::env::temp_dir(); - log_path.push("server_fixture.log"); + log_path.push(format!( + "server_fixture_{}_{}.log", + addrs.http_port, addrs.grpc_port + )); println!("****************"); println!("Server Logging to {:?}", log_path); @@ -144,8 +205,8 @@ impl TestServer { // Can enable for debbugging //.arg("-vv") .env("INFLUXDB_IOX_ID", "1") - .env("INFLUXDB_IOX_BIND_ADDR", HTTP_BIND_ADDR) - .env("INFLUXDB_IOX_GRPC_BIND_ADDR", GRPC_BIND_ADDR) + .env("INFLUXDB_IOX_BIND_ADDR", &addrs.http_bind_addr) + .env("INFLUXDB_IOX_GRPC_BIND_ADDR", &addrs.grpc_bind_addr) // redirect output to log file .stdout(stdout_log_file) .stderr(stderr_log_file) @@ -155,6 +216,7 @@ impl TestServer { Ok(Self { ready, dir, + addrs, server_process, }) } @@ -173,7 +235,7 @@ impl TestServer { Ok(()) } - async fn wait_until_ready(&self) { + async fn wait_until_ready(&self, initial_config: InitialConfig) { let mut ready = self.ready.lock().await; match *ready { ServerState::Started => {} // first time, need to try and start it @@ -217,7 +279,7 @@ impl TestServer { let try_http_connect = async { let client = reqwest::Client::new(); - let url = format!("{}/health", HTTP_BASE); + let url = format!("{}/health", self.addrs().http_base); let mut interval = tokio::time::interval(Duration::from_millis(500)); loop { match client.get(&url).send().await { @@ -250,16 +312,23 @@ impl TestServer { } } - // Set the writer id, if requested (TODO if requested) - let channel = self.grpc_channel().await.expect("gRPC should be running"); - let mut management_client = influxdb_iox_client::management::Client::new(channel); + // Set the writer id, if requested + match initial_config { + InitialConfig::SetWriterId => { + let channel = self.grpc_channel().await.expect("gRPC should be running"); + let mut management_client = influxdb_iox_client::management::Client::new(channel); + let id = NonZeroU32::new(42).expect("42 is non zero, among its other properties"); - let id = NonZeroU32::new(42).expect("42 is non zero, among its other properties"); - - management_client - .update_writer_id(id) - .await - .expect("set ID failed"); + management_client + .update_writer_id(id) + .await + .expect("set ID failed"); + println!("Set writer_id to {:?}", id); + } + InitialConfig::None => { + println!("Leaving database unconfigured"); + } + } } /// Create a connection channel for the gRPR endpoing @@ -267,20 +336,12 @@ impl TestServer { &self, ) -> influxdb_iox_client::connection::Result { influxdb_iox_client::connection::Builder::default() - .build(self.grpc_url_base()) + .build(&self.addrs().grpc_base) .await } - fn grpc_url_base(&self) -> &str { - GRPC_URL_BASE - } - - fn http_base(&self) -> &str { - HTTP_BASE - } - - fn iox_api_v1_base(&self) -> &str { - IOX_API_V1_BASE + fn addrs(&self) -> &BindAddresses { + &self.addrs } } @@ -289,8 +350,8 @@ impl std::fmt::Display for TestServer { write!( f, "TestServer (grpc {}, http {})", - self.grpc_url_base(), - self.http_base() + self.addrs().grpc_base, + self.addrs().http_base ) } } diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index 46c87a2535..f6fb718fa7 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -1,21 +1,9 @@ // The test in this file runs the server in a separate thread and makes HTTP // requests as a smoke test for the integration of the whole system. // -// As written, only one test of this style can run at a time. Add more data to -// the existing test to test more scenarios rather than adding more tests in the -// same style. +// The servers under test are managed using [`ServerFixture`] // -// Or, change the way this test behaves to create isolated instances by: -// -// - Finding an unused port for the server to run on and using that port in the -// URL -// - Creating a temporary directory for an isolated database path -// -// Or, change the tests to use one server and isolate through `org_id` by: -// -// - Starting one server before all the relevant tests are run -// - Creating a unique org_id per test -// - Stopping the server after all relevant tests are run +// Other rust tests are defined in the various submodules of end_to_end_cases use std::convert::TryInto; use std::str; @@ -36,6 +24,7 @@ type Error = Box; type Result = std::result::Result; pub mod common; + mod end_to_end_cases; use common::server_fixture::*; @@ -73,11 +62,6 @@ async fn read_and_write_data() { &mut storage_client, ) .await; - management_api::test(&mut management_client).await; - management_cli::test(&fixture).await; - write_cli::test(&fixture).await; - read_cli::test(&fixture).await; - test_http_error_messages(&influxdb2).await.unwrap(); } // TODO: Randomly generate org and bucket ids to ensure test data independence @@ -278,9 +262,11 @@ async fn write_data( Ok(()) } -// Don't make a separate #test function so that we can reuse the same -// server process -async fn test_http_error_messages(client: &influxdb2_client::Client) -> Result<()> { +#[tokio::test] +async fn test_http_error_messages() { + let server_fixture = ServerFixture::create_shared().await; + let client = server_fixture.influxdb2_client(); + // send malformed request (bucket id is invalid) let result = client .write_line_protocol("Bar", "Foo", "arbitrary") @@ -289,8 +275,6 @@ async fn test_http_error_messages(client: &influxdb2_client::Client) -> Result<( let expected_error = "HTTP request returned an error: 400 Bad Request, `{\"error\":\"Error parsing line protocol: A generic parsing error occurred: TakeWhile1\",\"error_code\":100}`"; assert_eq!(result.to_string(), expected_error); - - Ok(()) } /// substitutes "ns" --> ns_since_epoch, ns1-->ns_since_epoch+1, etc diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index ac3df4b1a4..ae21281858 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -4,15 +4,21 @@ use generated_types::google::protobuf::Empty; use generated_types::{google::protobuf::Duration, influxdata::iox::management::v1::*}; use influxdb_iox_client::management::{Client, CreateDatabaseError}; +use crate::common::server_fixture::ServerFixture; + use super::util::rand_name; -pub async fn test(client: &mut Client) { - test_list_update_remotes(client).await; - test_set_get_writer_id(client).await; - test_create_database_duplicate_name(client).await; - test_create_database_invalid_name(client).await; - test_list_databases(client).await; - test_create_get_database(client).await; +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_single_use().await; + let mut client = Client::new(server_fixture.grpc_channel()); + + test_list_update_remotes(&mut client).await; + test_set_get_writer_id(&mut client).await; + test_create_database_duplicate_name(&mut client).await; + test_create_database_invalid_name(&mut client).await; + test_list_databases(&mut client).await; + test_create_get_database(&mut client).await; } async fn test_list_update_remotes(client: &mut Client) { diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index c2b74d8a5a..c28c5fe56a 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -3,8 +3,10 @@ use predicates::prelude::*; use crate::common::server_fixture::ServerFixture; -pub async fn test(server_fixture: &ServerFixture) { - let addr = server_fixture.grpc_url_base(); +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_single_use().await; + let addr = server_fixture.grpc_base(); test_writer_id(addr).await; test_create_database(addr).await; diff --git a/tests/end_to_end_cases/read_cli.rs b/tests/end_to_end_cases/read_cli.rs index 5a5e8673ad..0c82973f98 100644 --- a/tests/end_to_end_cases/read_cli.rs +++ b/tests/end_to_end_cases/read_cli.rs @@ -6,9 +6,12 @@ use crate::common::server_fixture::ServerFixture; use super::util::rand_name; -pub async fn test(server_fixture: &ServerFixture) { +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_single_use().await; let db_name = rand_name(); - let addr = server_fixture.grpc_url_base(); + let addr = server_fixture.grpc_base(); + create_database(&db_name, addr).await; test_read_default(&db_name, addr).await; test_read_format_pretty(&db_name, addr).await; diff --git a/tests/end_to_end_cases/write_api.rs b/tests/end_to_end_cases/write_api.rs index 276395f4e9..9785e9e722 100644 --- a/tests/end_to_end_cases/write_api.rs +++ b/tests/end_to_end_cases/write_api.rs @@ -1,5 +1,3 @@ -use std::num::NonZeroU32; - use influxdb_iox_client::management::{self, generated_types::DatabaseRules}; use influxdb_iox_client::write::{self, WriteError}; use test_helpers::assert_contains; @@ -10,13 +8,10 @@ use crate::common::server_fixture::ServerFixture; #[tokio::test] async fn test_write() { - // TODO sort out changing the test ID let fixture = ServerFixture::create_shared().await; let mut management_client = management::Client::new(fixture.grpc_channel()); let mut write_client = write::Client::new(fixture.grpc_channel()); - const TEST_ID: u32 = 42; - let db_name = rand_name(); management_client @@ -27,11 +22,6 @@ async fn test_write() { .await .expect("create database failed"); - management_client - .update_writer_id(NonZeroU32::new(TEST_ID).unwrap()) - .await - .expect("set ID failed"); - let lp_lines = vec![ "cpu,region=west user=23.2 100", "cpu,region=west user=21.0 150", diff --git a/tests/end_to_end_cases/write_cli.rs b/tests/end_to_end_cases/write_cli.rs index a299d1b0ee..b3903c831f 100644 --- a/tests/end_to_end_cases/write_cli.rs +++ b/tests/end_to_end_cases/write_cli.rs @@ -6,9 +6,11 @@ use crate::common::server_fixture::ServerFixture; use super::util::rand_name; -pub async fn test(server_fixture: &ServerFixture) { +#[tokio::test] +async fn test() { + let server_fixture = ServerFixture::create_shared().await; let db_name = rand_name(); - let addr = server_fixture.grpc_url_base(); + let addr = server_fixture.grpc_base(); create_database(&db_name, addr).await; test_write(&db_name, addr).await; } From f568c083a4e68d6962f3c37e0bfb350cf383bb6a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 10 Mar 2021 09:25:27 -0500 Subject: [PATCH 016/104] fix: Do not leave child processes around after the end-to-end test (#955) --- Cargo.lock | 1 + Cargo.toml | 1 + tests/common/server_fixture.rs | 37 ++++++++++++++++++++++++++-------- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35d582dd7d..1a0895469a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1507,6 +1507,7 @@ dependencies = [ "opentelemetry-jaeger", "packers", "panic_logging", + "parking_lot", "predicates", "prost", "query", diff --git a/Cargo.toml b/Cargo.toml index 3c0f2f75ed..47a3118d39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,6 +94,7 @@ influxdb2_client = { path = "influxdb2_client" } influxdb_iox_client = { path = "influxdb_iox_client", features = ["flight"] } test_helpers = { path = "test_helpers" } once_cell = { version = "1.4.0", features = ["parking_lot"] } +parking_lot = "0.11.1" # Crates.io dependencies, in alphabetical order assert_cmd = "1.0.0" diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index a40644bc02..9ebb7c506e 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -1,7 +1,10 @@ use std::{ fs::File, str, - sync::atomic::{AtomicUsize, Ordering::SeqCst}, + sync::{ + atomic::{AtomicUsize, Ordering::SeqCst}, + Weak, + }, }; use std::{num::NonZeroU32, process::Child}; @@ -89,15 +92,33 @@ impl ServerFixture { /// This is currently implemented as a singleton so all tests *must* /// use a new database and not interfere with the existing database. pub async fn create_shared() -> Self { - static SERVER: OnceCell> = OnceCell::new(); + // Try and reuse the same shared server, if there is already + // one present + static SHARED_SERVER: OnceCell>> = OnceCell::new(); - let server = Arc::clone(SERVER.get_or_init(|| { - let server = TestServer::new().expect("Could start test server"); - Arc::new(server) - })); + let shared_server = SHARED_SERVER.get_or_init(|| parking_lot::Mutex::new(Weak::new())); + + let mut shared_server = shared_server.lock(); + + // is a shared server already present? + let server = match shared_server.upgrade() { + Some(server) => server, + None => { + // if not, create one + let server = TestServer::new().expect("Could start test server"); + let server = Arc::new(server); + + // ensure the server is ready + server.wait_until_ready(InitialConfig::SetWriterId).await; + // save a reference for other threads that may want to + // use this server, but don't prevent it from being + // destroyed when going out of scope + *shared_server = Arc::downgrade(&server); + server + } + }; + std::mem::drop(shared_server); - // ensure the server is ready - server.wait_until_ready(InitialConfig::SetWriterId).await; Self::create_common(server).await } From 2b7289002818ac31dec734e36be9e281aa442eab Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 10 Mar 2021 09:12:23 -0500 Subject: [PATCH 017/104] fix: Revert "fix: Shutdown stest fixture on test harness exit" This reverts commit 28fe808d7e0fbec3b6ea37a690251338a9091e90. Not needed after https://github.com/influxdata/influxdb_iox/pull/955 is merged --- scripts/cargo_bin_no_orphan.bash | 31 ------------------------- tests/common/mod.rs | 1 - tests/common/no_orphan_cargo.rs | 40 -------------------------------- tests/common/server_fixture.rs | 11 +++++---- 4 files changed, 7 insertions(+), 76 deletions(-) delete mode 100755 scripts/cargo_bin_no_orphan.bash delete mode 100644 tests/common/no_orphan_cargo.rs diff --git a/scripts/cargo_bin_no_orphan.bash b/scripts/cargo_bin_no_orphan.bash deleted file mode 100755 index ef80ca4e92..0000000000 --- a/scripts/cargo_bin_no_orphan.bash +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Runs a command and waits until it or the parent process dies. -# -# See https://github.com/influxdata/influxdb_iox/issues/951 -# -# Hack self destruction condition: -# when we stop using ServerFixture::create_shared and similar test setup techniques -# we can rely on RAII to properly tear down and remove this hack. -# - -parent=$(ps -o ppid= $$) - -cmd="$1" -shift - -trap 'kill $(jobs -p)' EXIT - -"${cmd}" "$@" & - -child="$!" - -echo child "${child}" >>/tmp/noorphan.log - -check_pid() { - kill -0 "$1" 2>/dev/null -} - -while check_pid "${parent}" && check_pid "${child}"; do - sleep 1 -done diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 1c96171864..a057eb4940 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,2 +1 @@ -mod no_orphan_cargo; pub mod server_fixture; diff --git a/tests/common/no_orphan_cargo.rs b/tests/common/no_orphan_cargo.rs deleted file mode 100644 index 1f0799bd05..0000000000 --- a/tests/common/no_orphan_cargo.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Like assert_cmd::cargo::CommandCargoExt -//! but with workaround for https://github.com/influxdata/influxdb_iox/issues/951 - -use std::env; -use std::path; -use std::process::Command; - -const WRAPPER: &str = "scripts/cargo_bin_no_orphan.bash"; - -/// Create a Command to run a specific binary of the current crate. -/// -/// The binary will be executed by a wrapper which ensures it gets -/// killed when the parent process (the caller) dies. -/// This is useful in test fixtures that lazily initialize background processes -/// in static Onces, which are never dropped and thus cannot be cleaned up with -/// RAII. -pub fn cargo_bin>(name: S) -> Command { - let mut cmd = Command::new(WRAPPER); - cmd.arg(cargo_bin_name(name.as_ref())); - cmd -} - -fn cargo_bin_name(name: &str) -> path::PathBuf { - target_dir().join(format!("{}{}", name, env::consts::EXE_SUFFIX)) -} - -// Adapted from -// https://github.com/rust-lang/cargo/blob/485670b3983b52289a2f353d589c57fae2f60f82/tests/testsuite/support/mod.rs#L507 -fn target_dir() -> path::PathBuf { - env::current_exe() - .ok() - .map(|mut path| { - path.pop(); - if path.ends_with("deps") { - path.pop(); - } - path - }) - .unwrap() -} diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 9ebb7c506e..084746d50a 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -1,14 +1,15 @@ +use assert_cmd::prelude::*; use std::{ fs::File, + num::NonZeroU32, + process::{Child, Command}, str, sync::{ atomic::{AtomicUsize, Ordering::SeqCst}, Weak, }, }; -use std::{num::NonZeroU32, process::Child}; -use crate::common::no_orphan_cargo::cargo_bin; use futures::prelude::*; use std::time::Duration; use tempfile::TempDir; @@ -222,7 +223,8 @@ impl TestServer { .expect("cloning file handle for stdout"); let stderr_log_file = log_file; - let server_process = cargo_bin("influxdb_iox") + let server_process = Command::cargo_bin("influxdb_iox") + .unwrap() // Can enable for debbugging //.arg("-vv") .env("INFLUXDB_IOX_ID", "1") @@ -246,7 +248,8 @@ impl TestServer { fn restart(&mut self) -> Result<()> { self.server_process.kill().unwrap(); self.server_process.wait().unwrap(); - self.server_process = cargo_bin("influxdb_iox") + self.server_process = Command::cargo_bin("influxdb_iox") + .unwrap() // Can enable for debbugging //.arg("-vv") .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) From 5434846250457d0aed5cb880fdc69c5a106805a9 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Wed, 10 Mar 2021 03:11:11 +0100 Subject: [PATCH 018/104] test: Detect if talking to the wrong test server Closes #952 A pragmatic fix for #952: since we already set the server id in `wait_until_ready`, let's start a test server without an ID (by not passing `INFLUXDB_IOX_ID`) and use the property of already having an ID as an indication that we're talking to a server instance that we didn't just start. It doesn't necessarily mean we're talking to the right server, but the main point of #952 was to avoid confusing error messages like "DatabaseAlreadyExists"; with this PR, the only way for that error to confuse developers is if we "unset" the writer ID of a server fixture and leave it there hanging, with in-memory side effects but no ID. Possible but unlikely, I think. --- tests/common/server_fixture.rs | 39 ++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 084746d50a..c64ea96fd9 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -225,9 +225,8 @@ impl TestServer { let server_process = Command::cargo_bin("influxdb_iox") .unwrap() - // Can enable for debbugging + // Can enable for debugging //.arg("-vv") - .env("INFLUXDB_IOX_ID", "1") .env("INFLUXDB_IOX_BIND_ADDR", &addrs.http_bind_addr) .env("INFLUXDB_IOX_GRPC_BIND_ADDR", &addrs.grpc_bind_addr) // redirect output to log file @@ -250,10 +249,9 @@ impl TestServer { self.server_process.wait().unwrap(); self.server_process = Command::cargo_bin("influxdb_iox") .unwrap() - // Can enable for debbugging + // Can enable for debugging //.arg("-vv") .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) - .env("INFLUXDB_IOX_ID", "1") .spawn() .unwrap(); Ok(()) @@ -336,23 +334,32 @@ impl TestServer { } } - // Set the writer id, if requested - match initial_config { + // Set the writer id, if requested; otherwise default to the default writer id: + // 1. + let id = match initial_config { InitialConfig::SetWriterId => { - let channel = self.grpc_channel().await.expect("gRPC should be running"); - let mut management_client = influxdb_iox_client::management::Client::new(channel); - let id = NonZeroU32::new(42).expect("42 is non zero, among its other properties"); - - management_client - .update_writer_id(id) - .await - .expect("set ID failed"); - println!("Set writer_id to {:?}", id); + NonZeroU32::new(42).expect("42 is non zero, among its other properties") } InitialConfig::None => { - println!("Leaving database unconfigured"); + NonZeroU32::new(1).expect("1 is non zero; the first one to be so, moreover") } + }; + + let channel = self.grpc_channel().await.expect("gRPC should be running"); + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + if let Ok(id) = management_client.get_writer_id().await { + // tell others that this server had some problem + *ready = ServerState::Error; + std::mem::drop(ready); + panic!("Server already has an ID ({}); possibly a stray/orphan server from another test run.", id); } + + management_client + .update_writer_id(id) + .await + .expect("set ID failed"); + println!("Set writer_id to {:?}", id); } /// Create a connection channel for the gRPR endpoing From 79105b2c50bbed1aa4e59dbb67117c8e810117ab Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 8 Mar 2021 14:40:40 +0100 Subject: [PATCH 019/104] feat: Make server no longer the default command --- Dockerfile | 3 +++ README.md | 6 +++--- docker/Dockerfile.iox | 2 ++ src/commands/server.rs | 2 +- src/main.rs | 11 ++++------- tests/common/server_fixture.rs | 2 ++ 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51079c774e..70b97a2dcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,3 +36,6 @@ COPY --from=build /root/influxdb_iox /usr/bin/influxdb_iox EXPOSE 8080 8082 ENTRYPOINT ["/usr/bin/influxdb_iox"] + +CMD ["server"] + diff --git a/README.md b/README.md index a576f50e58..6decf9931a 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ which will create a binary in `target/debug` that you can run with: You can compile and run with one command by using: ```shell -cargo run +cargo run -- server ``` When compiling for performance testing, build in release mode by using: @@ -175,13 +175,13 @@ cargo build --release which will create the corresponding binary in `target/release`: ```shell -./target/release/influxdb_iox +./target/release/influxdb_iox server ``` Similarly, you can do this in one step with: ```shell -cargo run --release +cargo run --release -- server ``` The server will, by default, start an HTTP API server on port `8080` and a gRPC server on port diff --git a/docker/Dockerfile.iox b/docker/Dockerfile.iox index e62d8f396d..8b1f33c2a2 100644 --- a/docker/Dockerfile.iox +++ b/docker/Dockerfile.iox @@ -20,3 +20,5 @@ COPY target/release/influxdb_iox /usr/bin/influxdb_iox EXPOSE 8080 8082 ENTRYPOINT ["influxdb_iox"] + +CMD ["server"] diff --git a/src/commands/server.rs b/src/commands/server.rs index 33ca6f2d0e..6484037e27 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -18,7 +18,7 @@ pub const FALLBACK_AWS_REGION: &str = "us-east-1"; #[derive(Debug, StructOpt)] #[structopt( name = "server", - about = "Runs in server mode (default)", + about = "Runs in server mode", long_about = "Run the IOx server.\n\nThe configuration options below can be \ set either with the command line flags or with the specified environment \ variable. If there is a file named '.env' in the current working directory, \ diff --git a/src/main.rs b/src/main.rs index d2064e3ce1..a46c4cc3d3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -87,6 +87,7 @@ struct Config { } #[derive(Debug, StructOpt)] +#[structopt(setting = structopt::clap::AppSettings::SubcommandRequiredElseHelp)] enum Command { /// Convert one storage format to another Convert { @@ -194,13 +195,9 @@ fn main() -> Result<(), std::io::Error> { } } None => { - // Note don't set up basic logging here, different logging rules apply in server - // mode - let res = influxdb_ioxd::main(logging_level, None).await; - if let Err(e) = res { - error!("Server shutdown with error: {}", e); - std::process::exit(ReturnCode::Failure as _); - } + unreachable!( + "SubcommandRequiredElseHelp will print help if there is no subcommand" + ); } } }); diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index c64ea96fd9..550fe41cd4 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -225,6 +225,7 @@ impl TestServer { let server_process = Command::cargo_bin("influxdb_iox") .unwrap() + .arg("server") // Can enable for debugging //.arg("-vv") .env("INFLUXDB_IOX_BIND_ADDR", &addrs.http_bind_addr) @@ -249,6 +250,7 @@ impl TestServer { self.server_process.wait().unwrap(); self.server_process = Command::cargo_bin("influxdb_iox") .unwrap() + .arg("server") // Can enable for debugging //.arg("-vv") .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) From ce3a36dd8bcb36e47656148f7706c10b902fbbb5 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Wed, 10 Mar 2021 17:26:42 +0100 Subject: [PATCH 020/104] chore: Simplify server command --- src/commands/server.rs | 100 ++++------------------------------------- src/influxdb_ioxd.rs | 7 +-- src/main.rs | 2 +- 3 files changed, 12 insertions(+), 97 deletions(-) diff --git a/src/commands/server.rs b/src/commands/server.rs index 6484037e27..1e93fb861e 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -214,23 +214,6 @@ Possible values (case insensitive): pub jaeger_host: Option, } -/// Load the config if `server` was not specified on the command line -/// (from environment variables and default) -/// -/// This pulls in config from the following sources, in order of precedence: -/// -/// - user set environment variables -/// - .env file contents -/// - pre-configured default values -pub fn load_config() -> Box { - // Load the Config struct - this pulls in any envs set by the user or - // sourced above, and applies any defaults. - // - - //let args = std::env::args().filter(|arg| arg != "server"); - Box::new(Config::from_iter(strip_server(std::env::args()).iter())) -} - fn parse_socket_addr(s: &str) -> std::io::Result { let mut addrs = s.to_socket_addrs()?; // when name resolution fails, to_socket_address returns a validation error @@ -241,24 +224,6 @@ fn parse_socket_addr(s: &str) -> std::io::Result { .expect("name resolution should return at least one address")) } -/// Strip everything prior to the "server" portion of the args so the generated -/// Clap instance plays nicely with the subcommand bits in main. -fn strip_server(args: impl Iterator) -> Vec { - let mut seen_server = false; - args.enumerate() - .filter_map(|(i, arg)| { - if i != 0 && !seen_server { - if arg == "server" { - seen_server = true; - } - None - } else { - Some(arg) - } - }) - .collect::>() -} - arg_enum! { #[derive(Debug, Copy, Clone, PartialEq)] pub enum ObjectStore { @@ -319,70 +284,23 @@ mod tests { use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; - #[test] - fn test_strip_server() { - assert_eq!( - strip_server(to_vec(&["cmd",]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "-v"]).into_iter()), - to_vec(&["cmd", "-v"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "-vv"]).into_iter()), - to_vec(&["cmd", "-vv"]) - ); - - // and it doesn't strip repeated instances of server - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "--gcp_path"]).into_iter()), - to_vec(&["cmd", "--gcp_path"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "--gcp_path", "server"]).into_iter()), - to_vec(&["cmd", "--gcp_path", "server"]) - ); - - assert_eq!( - strip_server(to_vec(&["cmd", "-vv"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-vv", "server"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-vv", "server", "-vv"]).into_iter()), - to_vec(&["cmd", "-vv"]) - ); - } - fn to_vec(v: &[&str]) -> Vec { v.iter().map(|s| s.to_string()).collect() } #[test] fn test_socketaddr() -> Result<(), clap::Error> { - let c = Config::from_iter_safe(strip_server( - to_vec(&["cmd", "server", "--api-bind", "127.0.0.1:1234"]).into_iter(), - ))?; + let c = Config::from_iter_safe( + to_vec(&["server", "--api-bind", "127.0.0.1:1234"]).into_iter(), + )?; assert_eq!( c.http_bind_address, SocketAddr::from(([127, 0, 0, 1], 1234)) ); - let c = Config::from_iter_safe(strip_server( - to_vec(&["cmd", "server", "--api-bind", "localhost:1234"]).into_iter(), - ))?; + let c = Config::from_iter_safe( + to_vec(&["server", "--api-bind", "localhost:1234"]).into_iter(), + )?; // depending on where the test runs, localhost will either resolve to a ipv4 or // an ipv6 addr. match c.http_bind_address { @@ -396,9 +314,9 @@ mod tests { }; assert_eq!( - Config::from_iter_safe(strip_server( - to_vec(&["cmd", "server", "--api-bind", "!@INv_a1d(ad0/resp_!"]).into_iter(), - )) + Config::from_iter_safe( + to_vec(&["server", "--api-bind", "!@INv_a1d(ad0/resp_!"]).into_iter(), + ) .map_err(|e| e.kind) .expect_err("must fail"), clap::ErrorKind::ValueValidation diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index dae3b6fedc..5ce1cb4c6c 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -1,6 +1,6 @@ use crate::commands::{ logging::LoggingLevel, - server::{load_config, Config, ObjectStore as ObjStoreOpt}, + server::{Config, ObjectStore as ObjStoreOpt}, }; use hyper::Server; use object_store::{ @@ -73,10 +73,7 @@ pub type Result = std::result::Result; /// /// The logging_level passed in is the global setting (e.g. if -v or /// -vv was passed in before 'server') -pub async fn main(logging_level: LoggingLevel, config: Option>) -> Result<()> { - // load config from environment if no command line - let config = config.unwrap_or_else(load_config); - +pub async fn main(logging_level: LoggingLevel, config: Box) -> Result<()> { // Handle the case if -v/-vv is specified both before and after the server // command let logging_level = logging_level.combine(LoggingLevel::new(config.verbose_count)); diff --git a/src/main.rs b/src/main.rs index a46c4cc3d3..633c85c5ae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -187,7 +187,7 @@ fn main() -> Result<(), std::io::Error> { Some(Command::Server(config)) => { // Note don't set up basic logging here, different logging rules apply in server // mode - let res = influxdb_ioxd::main(logging_level, Some(config)).await; + let res = influxdb_ioxd::main(logging_level, config).await; if let Err(e) = res { error!("Server shutdown with error: {}", e); From d2859a99d018a49a28f0a8083802053e4cdf0c81 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Wed, 10 Mar 2021 17:34:07 +0000 Subject: [PATCH 021/104] feat: add google longrunning operations stubs (#959) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .gitattributes | 2 + Cargo.lock | 4 +- buf.yaml | 9 +- generated_types/Cargo.toml | 2 +- generated_types/build.rs | 9 +- .../protos/google/api/annotations.proto | 31 ++ .../protos/google/api/client.proto | 99 +++++ generated_types/protos/google/api/http.proto | 375 ++++++++++++++++++ .../google/longrunning/operations.proto | 247 ++++++++++++ .../protos/google/rpc/error_details.proto | 251 ++++++++++++ .../protos/google/rpc/status.proto | 49 +++ generated_types/src/google.rs | 264 ++++++++++++ generated_types/src/lib.rs | 101 +++-- google_types/Cargo.toml | 3 - google_types/build.rs | 6 +- google_types/src/lib.rs | 259 ------------ 16 files changed, 1381 insertions(+), 330 deletions(-) create mode 100644 .gitattributes create mode 100644 generated_types/protos/google/api/annotations.proto create mode 100644 generated_types/protos/google/api/client.proto create mode 100644 generated_types/protos/google/api/http.proto create mode 100644 generated_types/protos/google/longrunning/operations.proto create mode 100644 generated_types/protos/google/rpc/error_details.proto create mode 100644 generated_types/protos/google/rpc/status.proto create mode 100644 generated_types/src/google.rs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..05eca1f076 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +generated_types/protos/google/ linguist-generated=true +generated_types/protos/grpc/ linguist-generated=true diff --git a/Cargo.lock b/Cargo.lock index 1a0895469a..50de29dc12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1203,6 +1203,7 @@ dependencies = [ "prost-types", "tonic", "tonic-build", + "tracing", ] [[package]] @@ -1255,9 +1256,6 @@ version = "0.1.0" dependencies = [ "prost", "prost-build", - "prost-types", - "tonic", - "tracing", ] [[package]] diff --git a/buf.yaml b/buf.yaml index fabedb23ea..e728e5f408 100644 --- a/buf.yaml +++ b/buf.yaml @@ -2,12 +2,13 @@ version: v1beta1 build: roots: - generated_types/protos/ - excludes: - - generated_types/protos/com - - generated_types/protos/influxdata/platform - - generated_types/protos/grpc lint: + ignore: + - google + - grpc + - com/github/influxdata/idpe/storage/read + - influxdata/platform use: - DEFAULT - STYLE_DEFAULT diff --git a/generated_types/Cargo.toml b/generated_types/Cargo.toml index f961c86610..3526ed5276 100644 --- a/generated_types/Cargo.toml +++ b/generated_types/Cargo.toml @@ -10,7 +10,7 @@ futures = "0.3.1" prost = "0.7" prost-types = "0.7" tonic = "0.4" - +tracing = "0.1" google_types = { path = "../google_types" } [build-dependencies] # In alphabetical order diff --git a/generated_types/build.rs b/generated_types/build.rs index c864c96eeb..74573e028d 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -29,7 +29,6 @@ fn generate_grpc_types(root: &Path) -> Result<()> { let idpe_path = root.join("com/github/influxdata/idpe/storage/read"); let management_path = root.join("influxdata/iox/management/v1"); let write_path = root.join("influxdata/iox/write/v1"); - let grpc_path = root.join("grpc/health/v1"); let proto_files = vec![ storage_path.join("test.proto"), @@ -42,7 +41,10 @@ fn generate_grpc_types(root: &Path) -> Result<()> { management_path.join("database_rules.proto"), management_path.join("service.proto"), write_path.join("service.proto"), - grpc_path.join("service.proto"), + root.join("grpc/health/v1/service.proto"), + root.join("google/longrunning/operations.proto"), + root.join("google/rpc/error_details.proto"), + root.join("google/rpc/status.proto"), ]; // Tell cargo to recompile if any of these proto files are changed @@ -54,7 +56,8 @@ fn generate_grpc_types(root: &Path) -> Result<()> { config .compile_well_known_types() - .extern_path(".google", "::google_types"); + .disable_comments(&[".google"]) + .extern_path(".google.protobuf", "::google_types::protobuf"); tonic_build::configure().compile_with_config(config, &proto_files, &[root.into()])?; diff --git a/generated_types/protos/google/api/annotations.proto b/generated_types/protos/google/api/annotations.proto new file mode 100644 index 0000000000..18dcf20990 --- /dev/null +++ b/generated_types/protos/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} \ No newline at end of file diff --git a/generated_types/protos/google/api/client.proto b/generated_types/protos/google/api/client.proto new file mode 100644 index 0000000000..2102623d30 --- /dev/null +++ b/generated_types/protos/google/api/client.proto @@ -0,0 +1,99 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "ClientProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // A definition of a client library method signature. + // + // In client libraries, each proto RPC corresponds to one or more methods + // which the end user is able to call, and calls the underlying RPC. + // Normally, this method receives a single argument (a struct or instance + // corresponding to the RPC request object). Defining this field will + // add one or more overloads providing flattened or simpler method signatures + // in some languages. + // + // The fields on the method signature are provided as a comma-separated + // string. + // + // For example, the proto RPC and annotation: + // + // rpc CreateSubscription(CreateSubscriptionRequest) + // returns (Subscription) { + // option (google.api.method_signature) = "name,topic"; + // } + // + // Would add the following Java overload (in addition to the method accepting + // the request object): + // + // public final Subscription createSubscription(String name, String topic) + // + // The following backwards-compatibility guidelines apply: + // + // * Adding this annotation to an unannotated method is backwards + // compatible. + // * Adding this annotation to a method which already has existing + // method signature annotations is backwards compatible if and only if + // the new method signature annotation is last in the sequence. + // * Modifying or removing an existing method signature annotation is + // a breaking change. + // * Re-ordering existing method signature annotations is a breaking + // change. + repeated string method_signature = 1051; +} + +extend google.protobuf.ServiceOptions { + // The hostname for this service. + // This should be specified with no prefix or protocol. + // + // Example: + // + // service Foo { + // option (google.api.default_host) = "foo.googleapi.com"; + // ... + // } + string default_host = 1049; + + // OAuth scopes needed for the client. + // + // Example: + // + // service Foo { + // option (google.api.oauth_scopes) = \ + // "https://www.googleapis.com/auth/cloud-platform"; + // ... + // } + // + // If there is more than one scope, use a comma-separated string: + // + // Example: + // + // service Foo { + // option (google.api.oauth_scopes) = \ + // "https://www.googleapis.com/auth/cloud-platform," + // "https://www.googleapis.com/auth/monitoring"; + // ... + // } + string oauth_scopes = 1050; +} diff --git a/generated_types/protos/google/api/http.proto b/generated_types/protos/google/api/http.proto new file mode 100644 index 0000000000..69460cf791 --- /dev/null +++ b/generated_types/protos/google/api/http.proto @@ -0,0 +1,375 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/generated_types/protos/google/longrunning/operations.proto b/generated_types/protos/google/longrunning/operations.proto new file mode 100644 index 0000000000..c1fdc6f529 --- /dev/null +++ b/generated_types/protos/google/longrunning/operations.proto @@ -0,0 +1,247 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.longrunning; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/rpc/status.proto"; +import "google/protobuf/descriptor.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.LongRunning"; +option go_package = "google.golang.org/genproto/googleapis/longrunning;longrunning"; +option java_multiple_files = true; +option java_outer_classname = "OperationsProto"; +option java_package = "com.google.longrunning"; +option php_namespace = "Google\\LongRunning"; + +extend google.protobuf.MethodOptions { + // Additional information regarding long-running operations. + // In particular, this specifies the types that are returned from + // long-running operations. + // + // Required for methods that return `google.longrunning.Operation`; invalid + // otherwise. + google.longrunning.OperationInfo operation_info = 1049; +} + +// Manages long-running operations with an API service. +// +// When an API method normally takes long time to complete, it can be designed +// to return [Operation][google.longrunning.Operation] to the client, and the client can use this +// interface to receive the real response asynchronously by polling the +// operation resource, or pass the operation resource to another API (such as +// Google Cloud Pub/Sub API) to receive the response. Any API service that +// returns long-running operations should implement the `Operations` interface +// so developers can have a consistent client experience. +service Operations { + option (google.api.default_host) = "longrunning.googleapis.com"; + + // Lists operations that match the specified filter in the request. If the + // server doesn't support this method, it returns `UNIMPLEMENTED`. + // + // NOTE: the `name` binding allows API services to override the binding + // to use different resource name schemes, such as `users/*/operations`. To + // override the binding, API services can add a binding such as + // `"/v1/{name=users/*}/operations"` to their service configuration. + // For backwards compatibility, the default name includes the operations + // collection id, however overriding users must ensure the name binding + // is the parent resource, without the operations collection id. + rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) { + option (google.api.http) = { + get: "/v1/{name=operations}" + }; + option (google.api.method_signature) = "name,filter"; + } + + // Gets the latest state of a long-running operation. Clients can use this + // method to poll the operation result at intervals as recommended by the API + // service. + rpc GetOperation(GetOperationRequest) returns (Operation) { + option (google.api.http) = { + get: "/v1/{name=operations/**}" + }; + option (google.api.method_signature) = "name"; + } + + // Deletes a long-running operation. This method indicates that the client is + // no longer interested in the operation result. It does not cancel the + // operation. If the server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. + rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=operations/**}" + }; + option (google.api.method_signature) = "name"; + } + + // Starts asynchronous cancellation on a long-running operation. The server + // makes a best effort to cancel the operation, but success is not + // guaranteed. If the server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. Clients can use + // [Operations.GetOperation][google.longrunning.Operations.GetOperation] or + // other methods to check whether the cancellation succeeded or whether the + // operation completed despite cancellation. On successful cancellation, + // the operation is not deleted; instead, it becomes an operation with + // an [Operation.error][google.longrunning.Operation.error] value with a [google.rpc.Status.code][google.rpc.Status.code] of 1, + // corresponding to `Code.CANCELLED`. + rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/{name=operations/**}:cancel" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Waits until the specified long-running operation is done or reaches at most + // a specified timeout, returning the latest state. If the operation is + // already done, the latest state is immediately returned. If the timeout + // specified is greater than the default HTTP/RPC timeout, the HTTP/RPC + // timeout is used. If the server does not support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. + // Note that this method is on a best-effort basis. It may return the latest + // state before the specified timeout (including immediately), meaning even an + // immediate response is no guarantee that the operation is done. + rpc WaitOperation(WaitOperationRequest) returns (Operation) { + } +} + +// This resource represents a long-running operation that is the result of a +// network API call. +message Operation { + // The server-assigned name, which is only unique within the same service that + // originally returns it. If you use the default HTTP mapping, the + // `name` should be a resource name ending with `operations/{unique_id}`. + string name = 1; + + // Service-specific metadata associated with the operation. It typically + // contains progress information and common metadata such as create time. + // Some services might not provide such metadata. Any method that returns a + // long-running operation should document the metadata type, if any. + google.protobuf.Any metadata = 2; + + // If the value is `false`, it means the operation is still in progress. + // If `true`, the operation is completed, and either `error` or `response` is + // available. + bool done = 3; + + // The operation result, which can be either an `error` or a valid `response`. + // If `done` == `false`, neither `error` nor `response` is set. + // If `done` == `true`, exactly one of `error` or `response` is set. + oneof result { + // The error result of the operation in case of failure or cancellation. + google.rpc.Status error = 4; + + // The normal response of the operation in case of success. If the original + // method returns no data on success, such as `Delete`, the response is + // `google.protobuf.Empty`. If the original method is standard + // `Get`/`Create`/`Update`, the response should be the resource. For other + // methods, the response should have the type `XxxResponse`, where `Xxx` + // is the original method name. For example, if the original method name + // is `TakeSnapshot()`, the inferred response type is + // `TakeSnapshotResponse`. + google.protobuf.Any response = 5; + } +} + +// The request message for [Operations.GetOperation][google.longrunning.Operations.GetOperation]. +message GetOperationRequest { + // The name of the operation resource. + string name = 1; +} + +// The request message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsRequest { + // The name of the operation's parent resource. + string name = 4; + + // The standard list filter. + string filter = 1; + + // The standard list page size. + int32 page_size = 2; + + // The standard list page token. + string page_token = 3; +} + +// The response message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsResponse { + // A list of operations that matches the specified filter in the request. + repeated Operation operations = 1; + + // The standard List next-page token. + string next_page_token = 2; +} + +// The request message for [Operations.CancelOperation][google.longrunning.Operations.CancelOperation]. +message CancelOperationRequest { + // The name of the operation resource to be cancelled. + string name = 1; +} + +// The request message for [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation]. +message DeleteOperationRequest { + // The name of the operation resource to be deleted. + string name = 1; +} + +// The request message for [Operations.WaitOperation][google.longrunning.Operations.WaitOperation]. +message WaitOperationRequest { + // The name of the operation resource to wait on. + string name = 1; + + // The maximum duration to wait before timing out. If left blank, the wait + // will be at most the time permitted by the underlying HTTP/RPC protocol. + // If RPC context deadline is also specified, the shorter one will be used. + google.protobuf.Duration timeout = 2; +} + +// A message representing the message types used by a long-running operation. +// +// Example: +// +// rpc LongRunningRecognize(LongRunningRecognizeRequest) +// returns (google.longrunning.Operation) { +// option (google.longrunning.operation_info) = { +// response_type: "LongRunningRecognizeResponse" +// metadata_type: "LongRunningRecognizeMetadata" +// }; +// } +message OperationInfo { + // Required. The message name of the primary return type for this + // long-running operation. + // This type will be used to deserialize the LRO's response. + // + // If the response is in a different package from the rpc, a fully-qualified + // message name must be used (e.g. `google.protobuf.Struct`). + // + // Note: Altering this value constitutes a breaking change. + string response_type = 1; + + // Required. The message name of the metadata type for this long-running + // operation. + // + // If the response is in a different package from the rpc, a fully-qualified + // message name must be used (e.g. `google.protobuf.Struct`). + // + // Note: Altering this value constitutes a breaking change. + string metadata_type = 2; +} diff --git a/generated_types/protos/google/rpc/error_details.proto b/generated_types/protos/google/rpc/error_details.proto new file mode 100644 index 0000000000..1dd1dffc8e --- /dev/null +++ b/generated_types/protos/google/rpc/error_details.proto @@ -0,0 +1,251 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// From https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto + +syntax = "proto3"; + +package google.rpc; + +import "google/protobuf/duration.proto"; + +option go_package = "google.golang.org/genproto/googleapis/rpc/errdetails;errdetails"; +option java_multiple_files = true; +option java_outer_classname = "ErrorDetailsProto"; +option java_package = "com.google.rpc"; +option objc_class_prefix = "RPC"; + +// Describes when the clients can retry a failed request. Clients could ignore +// the recommendation here or retry when this information is missing from error +// responses. +// +// It's always recommended that clients should use exponential backoff when +// retrying. +// +// Clients should wait until `retry_delay` amount of time has passed since +// receiving the error response before retrying. If retrying requests also +// fail, clients should use an exponential backoff scheme to gradually increase +// the delay between retries based on `retry_delay`, until either a maximum +// number of retries have been reached or a maximum retry delay cap has been +// reached. +message RetryInfo { + // Clients should wait at least this long between retrying the same request. + google.protobuf.Duration retry_delay = 1; +} + +// Describes additional debugging info. +message DebugInfo { + // The stack trace entries indicating where the error occurred. + repeated string stack_entries = 1; + + // Additional debugging information provided by the server. + string detail = 2; +} + +// Describes how a quota check failed. +// +// For example if a daily limit was exceeded for the calling project, +// a service could respond with a QuotaFailure detail containing the project +// id and the description of the quota limit that was exceeded. If the +// calling project hasn't enabled the service in the developer console, then +// a service could respond with the project id and set `service_disabled` +// to true. +// +// Also see RetryInfo and Help types for other details about handling a +// quota failure. +message QuotaFailure { + // A message type used to describe a single quota violation. For example, a + // daily quota or a custom quota that was exceeded. + message Violation { + // The subject on which the quota check failed. + // For example, "clientip:" or "project:". + string subject = 1; + + // A description of how the quota check failed. Clients can use this + // description to find more about the quota configuration in the service's + // public documentation, or find the relevant quota limit to adjust through + // developer console. + // + // For example: "Service disabled" or "Daily Limit for read operations + // exceeded". + string description = 2; + } + + // Describes all quota violations. + repeated Violation violations = 1; +} + +// Describes the cause of the error with structured details. +// +// Example of an error when contacting the "pubsub.googleapis.com" API when it +// is not enabled: +// +// { "reason": "API_DISABLED" +// "domain": "googleapis.com" +// "metadata": { +// "resource": "projects/123", +// "service": "pubsub.googleapis.com" +// } +// } +// +// This response indicates that the pubsub.googleapis.com API is not enabled. +// +// Example of an error that is returned when attempting to create a Spanner +// instance in a region that is out of stock: +// +// { "reason": "STOCKOUT" +// "domain": "spanner.googleapis.com", +// "metadata": { +// "availableRegions": "us-central1,us-east2" +// } +// } +message ErrorInfo { + // The reason of the error. This is a constant value that identifies the + // proximate cause of the error. Error reasons are unique within a particular + // domain of errors. This should be at most 63 characters and match + // /[A-Z0-9_]+/. + string reason = 1; + + // The logical grouping to which the "reason" belongs. The error domain + // is typically the registered service name of the tool or product that + // generates the error. Example: "pubsub.googleapis.com". If the error is + // generated by some common infrastructure, the error domain must be a + // globally unique value that identifies the infrastructure. For Google API + // infrastructure, the error domain is "googleapis.com". + string domain = 2; + + // Additional structured details about this error. + // + // Keys should match /[a-zA-Z0-9-_]/ and be limited to 64 characters in + // length. When identifying the current value of an exceeded limit, the units + // should be contained in the key, not the value. For example, rather than + // {"instanceLimit": "100/request"}, should be returned as, + // {"instanceLimitPerRequest": "100"}, if the client exceeds the number of + // instances that can be created in a single (batch) request. + map metadata = 3; +} + +// Describes what preconditions have failed. +// +// For example, if an RPC failed because it required the Terms of Service to be +// acknowledged, it could list the terms of service violation in the +// PreconditionFailure message. +message PreconditionFailure { + // A message type used to describe a single precondition failure. + message Violation { + // The type of PreconditionFailure. We recommend using a service-specific + // enum type to define the supported precondition violation subjects. For + // example, "TOS" for "Terms of Service violation". + string type = 1; + + // The subject, relative to the type, that failed. + // For example, "google.com/cloud" relative to the "TOS" type would indicate + // which terms of service is being referenced. + string subject = 2; + + // A description of how the precondition failed. Developers can use this + // description to understand how to fix the failure. + // + // For example: "Terms of service not accepted". + string description = 3; + } + + // Describes all precondition violations. + repeated Violation violations = 1; +} + +// Describes violations in a client request. This error type focuses on the +// syntactic aspects of the request. +message BadRequest { + // A message type used to describe a single bad request field. + message FieldViolation { + // A path leading to a field in the request body. The value will be a + // sequence of dot-separated identifiers that identify a protocol buffer + // field. E.g., "field_violations.field" would identify this field. + string field = 1; + + // A description of why the request element is bad. + string description = 2; + } + + // Describes all violations in a client request. + repeated FieldViolation field_violations = 1; +} + +// Contains metadata about the request that clients can attach when filing a bug +// or providing other forms of feedback. +message RequestInfo { + // An opaque string that should only be interpreted by the service generating + // it. For example, it can be used to identify requests in the service's logs. + string request_id = 1; + + // Any data that was used to serve this request. For example, an encrypted + // stack trace that can be sent back to the service provider for debugging. + string serving_data = 2; +} + +// Describes the resource that is being accessed. +message ResourceInfo { + // A name for the type of resource being accessed, e.g. "sql table", + // "cloud storage bucket", "file", "Google calendar"; or the type URL + // of the resource: e.g. "type.googleapis.com/google.pubsub.v1.Topic". + string resource_type = 1; + + // The name of the resource being accessed. For example, a shared calendar + // name: "example.com_4fghdhgsrgh@group.calendar.google.com", if the current + // error is [google.rpc.Code.PERMISSION_DENIED][google.rpc.Code.PERMISSION_DENIED]. + string resource_name = 2; + + // The owner of the resource (optional). + // For example, "user:" or "project:". + string owner = 3; + + // Describes what error is encountered when accessing this resource. + // For example, updating a cloud project may require the `writer` permission + // on the developer console project. + string description = 4; +} + +// Provides links to documentation or for performing an out of band action. +// +// For example, if a quota check failed with an error indicating the calling +// project hasn't enabled the accessed service, this can contain a URL pointing +// directly to the right place in the developer console to flip the bit. +message Help { + // Describes a URL link. + message Link { + // Describes what the link offers. + string description = 1; + + // The URL of the link. + string url = 2; + } + + // URL(s) pointing to additional information on handling the current error. + repeated Link links = 1; +} + +// Provides a localized error message that is safe to return to the user +// which can be attached to an RPC error. +message LocalizedMessage { + // The locale used following the specification defined at + // http://www.rfc-editor.org/rfc/bcp/bcp47.txt. + // Examples are: "en-US", "fr-CH", "es-MX" + string locale = 1; + + // The localized error message in the above locale. + string message = 2; +} \ No newline at end of file diff --git a/generated_types/protos/google/rpc/status.proto b/generated_types/protos/google/rpc/status.proto new file mode 100644 index 0000000000..ed7691ea20 --- /dev/null +++ b/generated_types/protos/google/rpc/status.proto @@ -0,0 +1,49 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// From https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto + +syntax = "proto3"; + +package google.rpc; + +import "google/protobuf/any.proto"; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/rpc/status;status"; +option java_multiple_files = true; +option java_outer_classname = "StatusProto"; +option java_package = "com.google.rpc"; +option objc_class_prefix = "RPC"; + +// The `Status` type defines a logical error model that is suitable for +// different programming environments, including REST APIs and RPC APIs. It is +// used by [gRPC](https://github.com/grpc). Each `Status` message contains +// three pieces of data: error code, error message, and error details. +// +// You can find out more about this error model and how to work with it in the +// [API Design Guide](https://cloud.google.com/apis/design/errors). +message Status { + // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + int32 code = 1; + + // A developer-facing error message, which should be in English. Any + // user-facing error message should be localized and sent in the + // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + string message = 2; + + // A list of messages that carry the error details. There is a common set of + // message types for APIs to use. + repeated google.protobuf.Any details = 3; +} \ No newline at end of file diff --git a/generated_types/src/google.rs b/generated_types/src/google.rs new file mode 100644 index 0000000000..51339d87bc --- /dev/null +++ b/generated_types/src/google.rs @@ -0,0 +1,264 @@ +//! Protobuf types for errors from the google standards and +//! conversions to `tonic::Status` + +pub use google_types::*; + +pub mod rpc { + include!(concat!(env!("OUT_DIR"), "/google.rpc.rs")); +} + +pub mod longrunning { + include!(concat!(env!("OUT_DIR"), "/google.longrunning.rs")); +} + +use self::protobuf::Any; +use prost::{ + bytes::{Bytes, BytesMut}, + Message, +}; +use std::convert::{TryFrom, TryInto}; +use std::iter::FromIterator; +use tonic::Status; +use tracing::error; + +// A newtype struct to provide conversion into tonic::Status +struct EncodeError(prost::EncodeError); + +impl From for tonic::Status { + fn from(error: EncodeError) -> Self { + error!(error=%error.0, "failed to serialise error response details"); + tonic::Status::unknown(format!("failed to serialise server error: {}", error.0)) + } +} + +impl From for EncodeError { + fn from(e: prost::EncodeError) -> Self { + Self(e) + } +} + +fn encode_status(code: tonic::Code, message: String, details: Any) -> tonic::Status { + let mut buffer = BytesMut::new(); + + let status = rpc::Status { + code: code as i32, + message: message.clone(), + details: vec![details], + }; + + match status.encode(&mut buffer) { + Ok(_) => tonic::Status::with_details(code, message, buffer.freeze()), + Err(e) => EncodeError(e).into(), + } +} + +/// Error returned if a request field has an invalid value. Includes +/// machinery to add parent field names for context -- thus it will +/// report `rules.write_timeout` than simply `write_timeout`. +#[derive(Debug, Default, Clone)] +pub struct FieldViolation { + pub field: String, + pub description: String, +} + +impl FieldViolation { + pub fn required(field: impl Into) -> Self { + Self { + field: field.into(), + description: "Field is required".to_string(), + } + } + + /// Re-scopes this error as the child of another field + pub fn scope(self, field: impl Into) -> Self { + let field = if self.field.is_empty() { + field.into() + } else { + [field.into(), self.field].join(".") + }; + + Self { + field, + description: self.description, + } + } +} + +fn encode_bad_request(violation: Vec) -> Result { + let mut buffer = BytesMut::new(); + + rpc::BadRequest { + field_violations: violation + .into_iter() + .map(|f| rpc::bad_request::FieldViolation { + field: f.field, + description: f.description, + }) + .collect(), + } + .encode(&mut buffer)?; + + Ok(Any { + type_url: "type.googleapis.com/google.rpc.BadRequest".to_string(), + value: buffer.freeze(), + }) +} + +impl From for tonic::Status { + fn from(f: FieldViolation) -> Self { + let message = format!("Violation for field \"{}\": {}", f.field, f.description); + + match encode_bad_request(vec![f]) { + Ok(details) => encode_status(tonic::Code::InvalidArgument, message, details), + Err(e) => e.into(), + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct InternalError {} + +impl From for tonic::Status { + fn from(_: InternalError) -> Self { + tonic::Status::new(tonic::Code::Internal, "Internal Error") + } +} + +#[derive(Debug, Default, Clone)] +pub struct AlreadyExists { + pub resource_type: String, + pub resource_name: String, + pub owner: String, + pub description: String, +} + +fn encode_resource_info( + resource_type: String, + resource_name: String, + owner: String, + description: String, +) -> Result { + let mut buffer = BytesMut::new(); + + rpc::ResourceInfo { + resource_type, + resource_name, + owner, + description, + } + .encode(&mut buffer)?; + + Ok(Any { + type_url: "type.googleapis.com/google.rpc.ResourceInfo".to_string(), + value: buffer.freeze(), + }) +} + +impl From for tonic::Status { + fn from(exists: AlreadyExists) -> Self { + let message = format!( + "Resource {}/{} already exists", + exists.resource_type, exists.resource_name + ); + match encode_resource_info( + exists.resource_type, + exists.resource_name, + exists.owner, + exists.description, + ) { + Ok(details) => encode_status(tonic::Code::AlreadyExists, message, details), + Err(e) => e.into(), + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct NotFound { + pub resource_type: String, + pub resource_name: String, + pub owner: String, + pub description: String, +} + +impl From for tonic::Status { + fn from(not_found: NotFound) -> Self { + let message = format!( + "Resource {}/{} not found", + not_found.resource_type, not_found.resource_name + ); + match encode_resource_info( + not_found.resource_type, + not_found.resource_name, + not_found.owner, + not_found.description, + ) { + Ok(details) => encode_status(tonic::Code::NotFound, message, details), + Err(e) => e.into(), + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct PreconditionViolation { + pub category: String, + pub subject: String, + pub description: String, +} + +fn encode_precondition_failure(violations: Vec) -> Result { + use rpc::precondition_failure::Violation; + + let mut buffer = BytesMut::new(); + + rpc::PreconditionFailure { + violations: violations + .into_iter() + .map(|x| Violation { + r#type: x.category, + subject: x.subject, + description: x.description, + }) + .collect(), + } + .encode(&mut buffer)?; + + Ok(Any { + type_url: "type.googleapis.com/google.rpc.PreconditionFailure".to_string(), + value: buffer.freeze(), + }) +} + +impl From for tonic::Status { + fn from(violation: PreconditionViolation) -> Self { + let message = format!( + "Precondition violation {} - {}: {}", + violation.subject, violation.category, violation.description + ); + match encode_precondition_failure(vec![violation]) { + Ok(details) => encode_status(tonic::Code::FailedPrecondition, message, details), + Err(e) => e.into(), + } + } +} + +/// An extension trait that adds the ability to convert an error +/// that can be converted to a String to a FieldViolation +pub trait FieldViolationExt { + type Output; + + fn field(self, field: &'static str) -> Result; +} + +impl FieldViolationExt for Result +where + E: ToString, +{ + type Output = T; + + fn field(self, field: &'static str) -> Result { + self.map_err(|e| FieldViolation { + field: field.to_string(), + description: e.to_string(), + }) + } +} diff --git a/generated_types/src/lib.rs b/generated_types/src/lib.rs index ce599dd48d..f06c7feaff 100644 --- a/generated_types/src/lib.rs +++ b/generated_types/src/lib.rs @@ -10,65 +10,63 @@ )] /// This module imports the generated protobuf code into a Rust module -/// heirarchy that matches the namespace heirarchy of the protobuf +/// hierarchy that matches the namespace hierarchy of the protobuf /// definitions -mod pb { - pub mod influxdata { - pub mod platform { - pub mod storage { - include!(concat!(env!("OUT_DIR"), "/influxdata.platform.storage.rs")); +pub mod influxdata { + pub mod platform { + pub mod storage { + include!(concat!(env!("OUT_DIR"), "/influxdata.platform.storage.rs")); - // Can't implement `Default` because `prost::Message` implements `Default` - impl TimestampRange { - pub fn max() -> Self { - TimestampRange { - start: std::i64::MIN, - end: std::i64::MAX, - } - } - } - } - } - - pub mod iox { - pub mod management { - pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/influxdata.iox.management.v1.rs")); - } - } - - pub mod write { - pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/influxdata.iox.write.v1.rs")); - } - } - } - } - - pub mod com { - pub mod github { - pub mod influxdata { - pub mod idpe { - pub mod storage { - pub mod read { - include!(concat!( - env!("OUT_DIR"), - "/com.github.influxdata.idpe.storage.read.rs" - )); - } + // Can't implement `Default` because `prost::Message` implements `Default` + impl TimestampRange { + pub fn max() -> Self { + TimestampRange { + start: std::i64::MIN, + end: std::i64::MAX, } } } } } - // Needed because of https://github.com/hyperium/tonic/issues/471 - pub mod grpc { - pub mod health { + pub mod iox { + pub mod management { pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/grpc.health.v1.rs")); + include!(concat!(env!("OUT_DIR"), "/influxdata.iox.management.v1.rs")); } } + + pub mod write { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/influxdata.iox.write.v1.rs")); + } + } + } +} + +pub mod com { + pub mod github { + pub mod influxdata { + pub mod idpe { + pub mod storage { + pub mod read { + include!(concat!( + env!("OUT_DIR"), + "/com.github.influxdata.idpe.storage.read.rs" + )); + } + } + } + } + } +} + +// Needed because of https://github.com/hyperium/tonic/issues/471 +pub mod grpc { + pub mod health { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/grpc.health.v1.rs")); + } } } @@ -81,8 +79,7 @@ pub const IOX_TESTING_SERVICE: &str = "influxdata.platform.storage.IOxTesting"; /// gRPC Arrow Flight Service pub const ARROW_SERVICE: &str = "arrow.flight.protocol.FlightService"; -pub use pb::com::github::influxdata::idpe::storage::read::*; -pub use pb::influxdata::platform::storage::*; +pub use com::github::influxdata::idpe::storage::read::*; +pub use influxdata::platform::storage::*; -pub use google_types as google; -pub use pb::{grpc, influxdata}; +pub mod google; diff --git a/google_types/Cargo.toml b/google_types/Cargo.toml index bfa9070899..ec72717045 100644 --- a/google_types/Cargo.toml +++ b/google_types/Cargo.toml @@ -7,9 +7,6 @@ edition = "2018" [dependencies] # In alphabetical order prost = "0.7" -prost-types = "0.7" -tonic = "0.4" -tracing = { version = "0.1" } [build-dependencies] # In alphabetical order prost-build = "0.7" diff --git a/google_types/build.rs b/google_types/build.rs index 5d30200d49..0112e26e84 100644 --- a/google_types/build.rs +++ b/google_types/build.rs @@ -9,11 +9,7 @@ type Result = std::result::Result; fn main() -> Result<()> { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("protos"); - let proto_files = vec![ - root.join("google/rpc/error_details.proto"), - root.join("google/rpc/status.proto"), - root.join("google/protobuf/types.proto"), - ]; + let proto_files = vec![root.join("google/protobuf/types.proto")]; // Tell cargo to recompile if any of these proto files are changed for proto_file in &proto_files { diff --git a/google_types/src/lib.rs b/google_types/src/lib.rs index e9c6ab1458..6a41539c3d 100644 --- a/google_types/src/lib.rs +++ b/google_types/src/lib.rs @@ -1,6 +1,3 @@ -//! Protobuf types for errors from the google standards and -//! conversions to `tonic::Status` - // This crate deliberately does not use the same linting rules as the other // crates because of all the generated code it contains that we don't have much // control over. @@ -39,263 +36,7 @@ mod pb { } } } - - pub mod rpc { - include!(concat!(env!("OUT_DIR"), "/google.rpc.rs")); - } } } pub use pb::google::*; - -use pb::google::protobuf::Any; -use prost::{ - bytes::{Bytes, BytesMut}, - Message, -}; -use std::convert::{TryFrom, TryInto}; -use std::iter::FromIterator; -use tonic::Status; -use tracing::error; - -// A newtype struct to provide conversion into tonic::Status -struct EncodeError(prost::EncodeError); - -impl From for tonic::Status { - fn from(error: EncodeError) -> Self { - error!(error=%error.0, "failed to serialise error response details"); - tonic::Status::unknown(format!("failed to serialise server error: {}", error.0)) - } -} - -impl From for EncodeError { - fn from(e: prost::EncodeError) -> Self { - Self(e) - } -} - -fn encode_status(code: tonic::Code, message: String, details: Any) -> tonic::Status { - let mut buffer = BytesMut::new(); - - let status = pb::google::rpc::Status { - code: code as i32, - message: message.clone(), - details: vec![details], - }; - - match status.encode(&mut buffer) { - Ok(_) => tonic::Status::with_details(code, message, buffer.freeze()), - Err(e) => EncodeError(e).into(), - } -} - -#[derive(Debug, Default, Clone)] -/// Error returned if a request field has an invalid value. Includes -/// machinery to add parent field names for context -- thus it will -/// report `rules.write_timeout` than simply `write_timeout`. -pub struct FieldViolation { - pub field: String, - pub description: String, -} - -impl FieldViolation { - pub fn required(field: impl Into) -> Self { - Self { - field: field.into(), - description: "Field is required".to_string(), - } - } - - /// Re-scopes this error as the child of another field - pub fn scope(self, field: impl Into) -> Self { - let field = if self.field.is_empty() { - field.into() - } else { - [field.into(), self.field].join(".") - }; - - Self { - field, - description: self.description, - } - } -} - -fn encode_bad_request(violation: Vec) -> Result { - let mut buffer = BytesMut::new(); - - pb::google::rpc::BadRequest { - field_violations: violation - .into_iter() - .map(|f| pb::google::rpc::bad_request::FieldViolation { - field: f.field, - description: f.description, - }) - .collect(), - } - .encode(&mut buffer)?; - - Ok(Any { - type_url: "type.googleapis.com/google.rpc.BadRequest".to_string(), - value: buffer.freeze(), - }) -} - -impl From for tonic::Status { - fn from(f: FieldViolation) -> Self { - let message = format!("Violation for field \"{}\": {}", f.field, f.description); - - match encode_bad_request(vec![f]) { - Ok(details) => encode_status(tonic::Code::InvalidArgument, message, details), - Err(e) => e.into(), - } - } -} - -#[derive(Debug, Default, Clone)] -pub struct InternalError {} - -impl From for tonic::Status { - fn from(_: InternalError) -> Self { - tonic::Status::new(tonic::Code::Internal, "Internal Error") - } -} - -#[derive(Debug, Default, Clone)] -pub struct AlreadyExists { - pub resource_type: String, - pub resource_name: String, - pub owner: String, - pub description: String, -} - -fn encode_resource_info( - resource_type: String, - resource_name: String, - owner: String, - description: String, -) -> Result { - let mut buffer = BytesMut::new(); - - pb::google::rpc::ResourceInfo { - resource_type, - resource_name, - owner, - description, - } - .encode(&mut buffer)?; - - Ok(Any { - type_url: "type.googleapis.com/google.rpc.ResourceInfo".to_string(), - value: buffer.freeze(), - }) -} - -impl From for tonic::Status { - fn from(exists: AlreadyExists) -> Self { - let message = format!( - "Resource {}/{} already exists", - exists.resource_type, exists.resource_name - ); - match encode_resource_info( - exists.resource_type, - exists.resource_name, - exists.owner, - exists.description, - ) { - Ok(details) => encode_status(tonic::Code::AlreadyExists, message, details), - Err(e) => e.into(), - } - } -} - -#[derive(Debug, Default, Clone)] -pub struct NotFound { - pub resource_type: String, - pub resource_name: String, - pub owner: String, - pub description: String, -} - -impl From for tonic::Status { - fn from(not_found: NotFound) -> Self { - let message = format!( - "Resource {}/{} not found", - not_found.resource_type, not_found.resource_name - ); - match encode_resource_info( - not_found.resource_type, - not_found.resource_name, - not_found.owner, - not_found.description, - ) { - Ok(details) => encode_status(tonic::Code::NotFound, message, details), - Err(e) => e.into(), - } - } -} - -#[derive(Debug, Default, Clone)] -pub struct PreconditionViolation { - pub category: String, - pub subject: String, - pub description: String, -} - -fn encode_precondition_failure(violations: Vec) -> Result { - use pb::google::rpc::precondition_failure::Violation; - - let mut buffer = BytesMut::new(); - - pb::google::rpc::PreconditionFailure { - violations: violations - .into_iter() - .map(|x| Violation { - r#type: x.category, - subject: x.subject, - description: x.description, - }) - .collect(), - } - .encode(&mut buffer)?; - - Ok(Any { - type_url: "type.googleapis.com/google.rpc.PreconditionFailure".to_string(), - value: buffer.freeze(), - }) -} - -impl From for tonic::Status { - fn from(violation: PreconditionViolation) -> Self { - let message = format!( - "Precondition violation {} - {}: {}", - violation.subject, violation.category, violation.description - ); - match encode_precondition_failure(vec![violation]) { - Ok(details) => encode_status(tonic::Code::FailedPrecondition, message, details), - Err(e) => e.into(), - } - } -} - -/// An extension trait that adds the ability to convert an error -/// that can be converted to a String to a FieldViolation -pub trait FieldViolationExt { - type Output; - - fn field(self, field: &'static str) -> Result; -} - -impl FieldViolationExt for Result -where - E: ToString, -{ - type Output = T; - - fn field(self, field: &'static str) -> Result { - self.map_err(|e| FieldViolation { - field: field.to_string(), - description: e.to_string(), - }) - } -} From 65972ccdfc4599e5c82163cc7958752c32973452 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Wed, 10 Mar 2021 18:05:39 +0000 Subject: [PATCH 022/104] feat: remove now redundant parts of the HTTP API (#931) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- data_types/src/http.rs | 6 - src/influxdb_ioxd/http.rs | 284 +------------------------------------- 2 files changed, 2 insertions(+), 288 deletions(-) diff --git a/data_types/src/http.rs b/data_types/src/http.rs index 2f595fb82a..904c2dc129 100644 --- a/data_types/src/http.rs +++ b/data_types/src/http.rs @@ -18,9 +18,3 @@ pub struct WalMetadataQuery { pub struct WalMetadataResponse { pub segments: Vec, } - -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -/// Body of the response to the /databases endpoint. -pub struct ListDatabasesResponse { - pub names: Vec, -} diff --git a/src/influxdb_ioxd/http.rs b/src/influxdb_ioxd/http.rs index 8e057f98e4..10edba3523 100644 --- a/src/influxdb_ioxd/http.rs +++ b/src/influxdb_ioxd/http.rs @@ -13,8 +13,7 @@ // Influx crates use arrow_deps::datafusion::physical_plan::collect; use data_types::{ - database_rules::DatabaseRules, - http::{ListDatabasesResponse, WalMetadataQuery}, + http::WalMetadataQuery, names::{org_and_bucket_to_database, OrgBucketMappingError}, DatabaseName, }; @@ -30,7 +29,7 @@ use futures::{self, StreamExt}; use http::header::{CONTENT_ENCODING, CONTENT_TYPE}; use hyper::{Body, Method, Request, Response, StatusCode}; use routerify::{prelude::*, Middleware, RequestInfo, Router, RouterError, RouterService}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use snafu::{OptionExt, ResultExt, Snafu}; use tracing::{debug, error, info}; @@ -314,15 +313,9 @@ where Ok(res) })) // this endpoint is for API backward compatibility with InfluxDB 2.x .post("/api/v2/write", write::) - .get("/ping", ping) .get("/health", health) - .get("/iox/api/v1/databases", list_databases::) - .put("/iox/api/v1/databases/:name", create_database::) - .get("/iox/api/v1/databases/:name", get_database::) .get("/iox/api/v1/databases/:name/query", query::) .get("/iox/api/v1/databases/:name/wal/meta", get_wal_meta::) - .put("/iox/api/v1/id", set_writer::) - .get("/iox/api/v1/id", get_writer::) .get("/api/v1/partitions", list_partitions::) .post("/api/v1/snapshot", snapshot_partition::) // Specify the error handler to handle any errors caused by @@ -532,69 +525,6 @@ async fn query( Ok(response) } -#[tracing::instrument(level = "debug")] -async fn list_databases(req: Request) -> Result, ApplicationError> -where - M: ConnectionManager + Send + Sync + Debug + 'static, -{ - let server = Arc::clone(&req.data::>>().expect("server state")); - - let names = server.db_names_sorted().await; - let json = serde_json::to_string(&ListDatabasesResponse { names }) - .context(InternalSerializationError)?; - Ok(Response::new(Body::from(json))) -} - -#[tracing::instrument(level = "debug")] -async fn create_database( - req: Request, -) -> Result, ApplicationError> { - let server = Arc::clone(&req.data::>>().expect("server state")); - - // with routerify, we shouldn't have gotten here without this being set - let db_name = req - .param("name") - .expect("db name must have been set") - .clone(); - let body = parse_body(req).await?; - - let rules: DatabaseRules = serde_json::from_slice(body.as_ref()).context(InvalidRequestBody)?; - - server - .create_database(db_name, rules) - .await - .context(ErrorCreatingDatabase)?; - - Ok(Response::new(Body::empty())) -} - -#[tracing::instrument(level = "debug")] -async fn get_database( - req: Request, -) -> Result, ApplicationError> { - let server = Arc::clone(&req.data::>>().expect("server state")); - - // with routerify, we shouldn't have gotten here without this being set - let db_name_str = req - .param("name") - .expect("db name must have been set") - .clone(); - let db_name = DatabaseName::new(&db_name_str).context(DatabaseNameError)?; - let db = server - .db_rules(&db_name) - .await - .context(DatabaseNotFound { name: &db_name_str })?; - - let data = serde_json::to_string(&db).context(JsonGenerationError)?; - let response = Response::builder() - .header("Content-Type", "application/json") - .status(StatusCode::OK) - .body(Body::from(data)) - .expect("builder should be successful"); - - Ok(response) -} - #[tracing::instrument(level = "debug")] async fn get_wal_meta( req: Request, @@ -652,73 +582,6 @@ async fn get_wal_meta( Ok(response) } -#[tracing::instrument(level = "debug")] -async fn set_writer( - req: Request, -) -> Result, ApplicationError> { - let server = Arc::clone(&req.data::>>().expect("server state")); - - // Read the request body - let body = parse_body(req).await?; - - // Parse the JSON body into a structure - #[derive(Serialize, Deserialize)] - struct WriterIdBody { - id: u32, - } - let req: WriterIdBody = serde_json::from_slice(body.as_ref()).context(InvalidRequestBody)?; - - // Set the writer ID - server.set_id(req.id); - - // Build a HTTP 200 response - let response = Response::builder() - .status(StatusCode::OK) - .body(Body::from( - serde_json::to_string(&req).expect("json encoding should not fail"), - )) - .expect("builder should be successful"); - - Ok(response) -} - -#[tracing::instrument(level = "debug")] -async fn get_writer( - req: Request, -) -> Result, ApplicationError> { - let id = { - let server = Arc::clone(&req.data::>>().expect("server state")); - server.require_id() - }; - - // Parse the JSON body into a structure - #[derive(Serialize)] - struct WriterIdBody { - id: u32, - } - - let body = WriterIdBody { - id: id.unwrap_or(0), - }; - - // Build a HTTP 200 response - let response = Response::builder() - .status(StatusCode::OK) - .body(Body::from( - serde_json::to_string(&body).expect("json encoding should not fail"), - )) - .expect("builder should be successful"); - - Ok(response) -} - -// Route to test that the server is alive -#[tracing::instrument(level = "debug")] -async fn ping(_: Request) -> Result, ApplicationError> { - let response_body = "PONG"; - Ok(Response::new(Body::from(response_body.to_string()))) -} - #[tracing::instrument(level = "debug")] async fn health(_: Request) -> Result, ApplicationError> { let response_body = "OK"; @@ -849,22 +712,6 @@ mod tests { type Error = Box; type Result = std::result::Result; - #[tokio::test] - async fn test_ping() -> Result<()> { - let test_storage = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - let server_url = test_server(Arc::clone(&test_storage)); - - let client = Client::new(); - let response = client.get(&format!("{}/ping", server_url)).send().await; - - // Print the response so if the test fails, we have a log of what went wrong - check_response("ping", response, StatusCode::OK, "PONG").await; - Ok(()) - } - #[tokio::test] async fn test_health() -> Result<()> { let test_storage = Arc::new(AppServer::new( @@ -1123,133 +970,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn set_writer_id() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let data = r#"{"id":42}"#; - - let client = Client::new(); - - let response = client - .put(&format!("{}/iox/api/v1/id", server_url)) - .body(data) - .send() - .await; - - check_response("set_writer_id", response, StatusCode::OK, data).await; - - assert_eq!(server.require_id().expect("should be set"), 42); - - // Check get_writer_id - let response = client - .get(&format!("{}/iox/api/v1/id", server_url)) - .send() - .await; - - check_response("get_writer_id", response, StatusCode::OK, data).await; - } - - #[tokio::test] - async fn list_databases() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let database_names: Vec = vec!["foo_bar", "foo_baz"] - .iter() - .map(|i| i.to_string()) - .collect(); - - for database_name in &database_names { - let rules = DatabaseRules { - name: database_name.clone(), - ..Default::default() - }; - server.create_database(database_name, rules).await.unwrap(); - } - - let client = Client::new(); - let response = client - .get(&format!("{}/iox/api/v1/databases", server_url)) - .send() - .await; - - let data = serde_json::to_string(&ListDatabasesResponse { - names: database_names, - }) - .unwrap(); - check_response("list_databases", response, StatusCode::OK, &data).await; - } - - #[tokio::test] - async fn create_database() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let data = r#"{}"#; - - let database_name = DatabaseName::new("foo_bar").unwrap(); - - let client = Client::new(); - let response = client - .put(&format!( - "{}/iox/api/v1/databases/{}", - server_url, database_name - )) - .body(data) - .send() - .await; - - check_response("create_database", response, StatusCode::OK, "").await; - - server.db(&database_name).await.unwrap(); - let db_rules = server.db_rules(&database_name).await.unwrap(); - assert!(db_rules.mutable_buffer_config.is_some()); - } - - #[tokio::test] - async fn get_database() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let database_name = "foo_bar"; - let rules = DatabaseRules { - name: database_name.to_owned(), - ..Default::default() - }; - let data = serde_json::to_string(&rules).unwrap(); - - server.create_database(database_name, rules).await.unwrap(); - - let client = Client::new(); - let response = client - .get(&format!( - "{}/iox/api/v1/databases/{}", - server_url, database_name - )) - .send() - .await; - - check_response("get_database", response, StatusCode::OK, &data).await; - } - #[tokio::test] async fn get_wal_meta() { let server = Arc::new(AppServer::new( From b8942dce0d8bc2bd692c1726f2993b8cebbfa0d4 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Wed, 10 Mar 2021 18:47:56 +0000 Subject: [PATCH 023/104] fix: remove unused proto files (#963) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .../protos/google/rpc/error_details.proto | 251 ------------------ google_types/protos/google/rpc/status.proto | 49 ---- 2 files changed, 300 deletions(-) delete mode 100644 google_types/protos/google/rpc/error_details.proto delete mode 100644 google_types/protos/google/rpc/status.proto diff --git a/google_types/protos/google/rpc/error_details.proto b/google_types/protos/google/rpc/error_details.proto deleted file mode 100644 index 1dd1dffc8e..0000000000 --- a/google_types/protos/google/rpc/error_details.proto +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// From https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto - -syntax = "proto3"; - -package google.rpc; - -import "google/protobuf/duration.proto"; - -option go_package = "google.golang.org/genproto/googleapis/rpc/errdetails;errdetails"; -option java_multiple_files = true; -option java_outer_classname = "ErrorDetailsProto"; -option java_package = "com.google.rpc"; -option objc_class_prefix = "RPC"; - -// Describes when the clients can retry a failed request. Clients could ignore -// the recommendation here or retry when this information is missing from error -// responses. -// -// It's always recommended that clients should use exponential backoff when -// retrying. -// -// Clients should wait until `retry_delay` amount of time has passed since -// receiving the error response before retrying. If retrying requests also -// fail, clients should use an exponential backoff scheme to gradually increase -// the delay between retries based on `retry_delay`, until either a maximum -// number of retries have been reached or a maximum retry delay cap has been -// reached. -message RetryInfo { - // Clients should wait at least this long between retrying the same request. - google.protobuf.Duration retry_delay = 1; -} - -// Describes additional debugging info. -message DebugInfo { - // The stack trace entries indicating where the error occurred. - repeated string stack_entries = 1; - - // Additional debugging information provided by the server. - string detail = 2; -} - -// Describes how a quota check failed. -// -// For example if a daily limit was exceeded for the calling project, -// a service could respond with a QuotaFailure detail containing the project -// id and the description of the quota limit that was exceeded. If the -// calling project hasn't enabled the service in the developer console, then -// a service could respond with the project id and set `service_disabled` -// to true. -// -// Also see RetryInfo and Help types for other details about handling a -// quota failure. -message QuotaFailure { - // A message type used to describe a single quota violation. For example, a - // daily quota or a custom quota that was exceeded. - message Violation { - // The subject on which the quota check failed. - // For example, "clientip:" or "project:". - string subject = 1; - - // A description of how the quota check failed. Clients can use this - // description to find more about the quota configuration in the service's - // public documentation, or find the relevant quota limit to adjust through - // developer console. - // - // For example: "Service disabled" or "Daily Limit for read operations - // exceeded". - string description = 2; - } - - // Describes all quota violations. - repeated Violation violations = 1; -} - -// Describes the cause of the error with structured details. -// -// Example of an error when contacting the "pubsub.googleapis.com" API when it -// is not enabled: -// -// { "reason": "API_DISABLED" -// "domain": "googleapis.com" -// "metadata": { -// "resource": "projects/123", -// "service": "pubsub.googleapis.com" -// } -// } -// -// This response indicates that the pubsub.googleapis.com API is not enabled. -// -// Example of an error that is returned when attempting to create a Spanner -// instance in a region that is out of stock: -// -// { "reason": "STOCKOUT" -// "domain": "spanner.googleapis.com", -// "metadata": { -// "availableRegions": "us-central1,us-east2" -// } -// } -message ErrorInfo { - // The reason of the error. This is a constant value that identifies the - // proximate cause of the error. Error reasons are unique within a particular - // domain of errors. This should be at most 63 characters and match - // /[A-Z0-9_]+/. - string reason = 1; - - // The logical grouping to which the "reason" belongs. The error domain - // is typically the registered service name of the tool or product that - // generates the error. Example: "pubsub.googleapis.com". If the error is - // generated by some common infrastructure, the error domain must be a - // globally unique value that identifies the infrastructure. For Google API - // infrastructure, the error domain is "googleapis.com". - string domain = 2; - - // Additional structured details about this error. - // - // Keys should match /[a-zA-Z0-9-_]/ and be limited to 64 characters in - // length. When identifying the current value of an exceeded limit, the units - // should be contained in the key, not the value. For example, rather than - // {"instanceLimit": "100/request"}, should be returned as, - // {"instanceLimitPerRequest": "100"}, if the client exceeds the number of - // instances that can be created in a single (batch) request. - map metadata = 3; -} - -// Describes what preconditions have failed. -// -// For example, if an RPC failed because it required the Terms of Service to be -// acknowledged, it could list the terms of service violation in the -// PreconditionFailure message. -message PreconditionFailure { - // A message type used to describe a single precondition failure. - message Violation { - // The type of PreconditionFailure. We recommend using a service-specific - // enum type to define the supported precondition violation subjects. For - // example, "TOS" for "Terms of Service violation". - string type = 1; - - // The subject, relative to the type, that failed. - // For example, "google.com/cloud" relative to the "TOS" type would indicate - // which terms of service is being referenced. - string subject = 2; - - // A description of how the precondition failed. Developers can use this - // description to understand how to fix the failure. - // - // For example: "Terms of service not accepted". - string description = 3; - } - - // Describes all precondition violations. - repeated Violation violations = 1; -} - -// Describes violations in a client request. This error type focuses on the -// syntactic aspects of the request. -message BadRequest { - // A message type used to describe a single bad request field. - message FieldViolation { - // A path leading to a field in the request body. The value will be a - // sequence of dot-separated identifiers that identify a protocol buffer - // field. E.g., "field_violations.field" would identify this field. - string field = 1; - - // A description of why the request element is bad. - string description = 2; - } - - // Describes all violations in a client request. - repeated FieldViolation field_violations = 1; -} - -// Contains metadata about the request that clients can attach when filing a bug -// or providing other forms of feedback. -message RequestInfo { - // An opaque string that should only be interpreted by the service generating - // it. For example, it can be used to identify requests in the service's logs. - string request_id = 1; - - // Any data that was used to serve this request. For example, an encrypted - // stack trace that can be sent back to the service provider for debugging. - string serving_data = 2; -} - -// Describes the resource that is being accessed. -message ResourceInfo { - // A name for the type of resource being accessed, e.g. "sql table", - // "cloud storage bucket", "file", "Google calendar"; or the type URL - // of the resource: e.g. "type.googleapis.com/google.pubsub.v1.Topic". - string resource_type = 1; - - // The name of the resource being accessed. For example, a shared calendar - // name: "example.com_4fghdhgsrgh@group.calendar.google.com", if the current - // error is [google.rpc.Code.PERMISSION_DENIED][google.rpc.Code.PERMISSION_DENIED]. - string resource_name = 2; - - // The owner of the resource (optional). - // For example, "user:" or "project:". - string owner = 3; - - // Describes what error is encountered when accessing this resource. - // For example, updating a cloud project may require the `writer` permission - // on the developer console project. - string description = 4; -} - -// Provides links to documentation or for performing an out of band action. -// -// For example, if a quota check failed with an error indicating the calling -// project hasn't enabled the accessed service, this can contain a URL pointing -// directly to the right place in the developer console to flip the bit. -message Help { - // Describes a URL link. - message Link { - // Describes what the link offers. - string description = 1; - - // The URL of the link. - string url = 2; - } - - // URL(s) pointing to additional information on handling the current error. - repeated Link links = 1; -} - -// Provides a localized error message that is safe to return to the user -// which can be attached to an RPC error. -message LocalizedMessage { - // The locale used following the specification defined at - // http://www.rfc-editor.org/rfc/bcp/bcp47.txt. - // Examples are: "en-US", "fr-CH", "es-MX" - string locale = 1; - - // The localized error message in the above locale. - string message = 2; -} \ No newline at end of file diff --git a/google_types/protos/google/rpc/status.proto b/google_types/protos/google/rpc/status.proto deleted file mode 100644 index ed7691ea20..0000000000 --- a/google_types/protos/google/rpc/status.proto +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// From https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto - -syntax = "proto3"; - -package google.rpc; - -import "google/protobuf/any.proto"; - -option cc_enable_arenas = true; -option go_package = "google.golang.org/genproto/googleapis/rpc/status;status"; -option java_multiple_files = true; -option java_outer_classname = "StatusProto"; -option java_package = "com.google.rpc"; -option objc_class_prefix = "RPC"; - -// The `Status` type defines a logical error model that is suitable for -// different programming environments, including REST APIs and RPC APIs. It is -// used by [gRPC](https://github.com/grpc). Each `Status` message contains -// three pieces of data: error code, error message, and error details. -// -// You can find out more about this error model and how to work with it in the -// [API Design Guide](https://cloud.google.com/apis/design/errors). -message Status { - // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. - int32 code = 1; - - // A developer-facing error message, which should be in English. Any - // user-facing error message should be localized and sent in the - // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. - string message = 2; - - // A list of messages that carry the error details. There is a common set of - // message types for APIs to use. - repeated google.protobuf.Any details = 3; -} \ No newline at end of file From d1a8872de646a53248f93897430b0811f0a0a8c0 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Wed, 10 Mar 2021 01:03:41 +0100 Subject: [PATCH 024/104] refactor: Turn server in a proper command Turn `server` into proper command, with a `command` function in its own module, its own error type, etc, in preparation of adding subcommands to it. --- src/commands/server.rs | 15 +++++++++++++++ src/main.rs | 10 ++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/commands/server.rs b/src/commands/server.rs index 1e93fb861e..9afaa13ed0 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -1,9 +1,12 @@ //! Implementation of command line option for manipulating and showing server //! config +use crate::commands::logging::LoggingLevel; +use crate::influxdb_ioxd; use clap::arg_enum; use std::{net::SocketAddr, net::ToSocketAddrs, path::PathBuf}; use structopt::StructOpt; +use thiserror::Error; /// The default bind address for the HTTP API. pub const DEFAULT_API_BIND_ADDR: &str = "127.0.0.1:8080"; @@ -15,6 +18,14 @@ pub const DEFAULT_GRPC_BIND_ADDR: &str = "127.0.0.1:8082"; /// specified. pub const FALLBACK_AWS_REGION: &str = "us-east-1"; +#[derive(Debug, Error)] +pub enum Error { + #[error("Server error")] + ServerError(#[from] influxdb_ioxd::Error), +} + +pub type Result = std::result::Result; + #[derive(Debug, StructOpt)] #[structopt( name = "server", @@ -214,6 +225,10 @@ Possible values (case insensitive): pub jaeger_host: Option, } +pub async fn command(logging_level: LoggingLevel, config: Box) -> Result<()> { + Ok(influxdb_ioxd::main(logging_level, config).await?) +} + fn parse_socket_addr(s: &str) -> std::io::Result { let mut addrs = s.to_socket_addrs()?; // when name resolution fails, to_socket_address returns a validation error diff --git a/src/main.rs b/src/main.rs index 633c85c5ae..7fe7c2d06c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,7 @@ use std::str::FromStr; use dotenv::dotenv; use structopt::StructOpt; use tokio::runtime::Runtime; -use tracing::{debug, error, warn}; +use tracing::{debug, warn}; use commands::logging::LoggingLevel; use ingest::parquet::writer::CompressionLevel; @@ -187,11 +187,9 @@ fn main() -> Result<(), std::io::Error> { Some(Command::Server(config)) => { // Note don't set up basic logging here, different logging rules apply in server // mode - let res = influxdb_ioxd::main(logging_level, config).await; - - if let Err(e) = res { - error!("Server shutdown with error: {}", e); - std::process::exit(ReturnCode::Failure as _); + if let Err(e) = commands::server::command(logging_level, config).await { + eprintln!("Server command failed: {}", e); + std::process::exit(ReturnCode::Failure as _) } } None => { From af553f3b38eb37863aa7518ea9d57ac05d9b38a8 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 11 Mar 2021 06:14:49 -0500 Subject: [PATCH 025/104] refactor: run some more end to end tests as tokio tests (#964) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- tests/end-to-end.rs | 9 ------- tests/end_to_end_cases/storage_api.rs | 38 +++++++++++++++------------ 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index f6fb718fa7..c5f0e8f7bc 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -53,15 +53,6 @@ async fn read_and_write_data() { storage_api::test(&mut storage_client, &scenario).await; flight_api::test(&fixture, &scenario, sql_query, &expected_read_data).await; } - - // These tests manage their own data - storage_api::read_group_test(&mut management_client, &influxdb2, &mut storage_client).await; - storage_api::read_window_aggregate_test( - &mut management_client, - &influxdb2, - &mut storage_client, - ) - .await; } // TODO: Randomly generate org and bucket ids to ensure test data independence diff --git a/tests/end_to_end_cases/storage_api.rs b/tests/end_to_end_cases/storage_api.rs index dbd74f6441..b547650d71 100644 --- a/tests/end_to_end_cases/storage_api.rs +++ b/tests/end_to_end_cases/storage_api.rs @@ -1,4 +1,4 @@ -use crate::{create_database, substitute_nanos, Scenario}; +use crate::{common::server_fixture::ServerFixture, create_database, substitute_nanos, Scenario}; use futures::prelude::*; use generated_types::{ aggregate::AggregateType, @@ -281,25 +281,27 @@ async fn measurement_fields_endpoint( assert_eq!(field.timestamp, scenario.ns_since_epoch() + 4); } -pub async fn read_group_test( - management: &mut management::Client, - influxdb2: &influxdb2_client::Client, - storage_client: &mut StorageClient, -) { +#[tokio::test] +pub async fn read_group_test() { + let fixture = ServerFixture::create_shared().await; + let mut management = management::Client::new(fixture.grpc_channel()); + let mut storage_client = StorageClient::new(fixture.grpc_channel()); + let influxdb2 = fixture.influxdb2_client(); + let scenario = Scenario::default() .set_org_id("0000111100001110") .set_bucket_id("1111000011110001"); - create_database(management, &scenario.database_name()).await; + create_database(&mut management, &scenario.database_name()).await; load_read_group_data(&influxdb2, &scenario).await; let read_source = scenario.read_source(); - test_read_group_none_agg(storage_client, &read_source).await; - test_read_group_none_agg_with_predicate(storage_client, &read_source).await; - test_read_group_sum_agg(storage_client, &read_source).await; - test_read_group_last_agg(storage_client, &read_source).await; + test_read_group_none_agg(&mut storage_client, &read_source).await; + test_read_group_none_agg_with_predicate(&mut storage_client, &read_source).await; + test_read_group_sum_agg(&mut storage_client, &read_source).await; + test_read_group_last_agg(&mut storage_client, &read_source).await; } async fn load_read_group_data(client: &influxdb2_client::Client, scenario: &Scenario) { @@ -534,17 +536,19 @@ async fn test_read_group_last_agg( } // Standalone test that all the pipes are hooked up for read window aggregate -pub async fn read_window_aggregate_test( - management: &mut management::Client, - influxdb2: &influxdb2_client::Client, - storage_client: &mut StorageClient, -) { +#[tokio::test] +pub async fn read_window_aggregate_test() { + let fixture = ServerFixture::create_shared().await; + let mut management = management::Client::new(fixture.grpc_channel()); + let mut storage_client = StorageClient::new(fixture.grpc_channel()); + let influxdb2 = fixture.influxdb2_client(); + let scenario = Scenario::default() .set_org_id("0000111100001100") .set_bucket_id("1111000011110011"); let read_source = scenario.read_source(); - create_database(management, &scenario.database_name()).await; + create_database(&mut management, &scenario.database_name()).await; let line_protocol = vec![ "h2o,state=MA,city=Boston temp=70.0 100", From 0ff527285c9e016b356f1eedcfad4af062996811 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Thu, 11 Mar 2021 11:33:53 +0000 Subject: [PATCH 026/104] refactor: remove unnecessary async from DatabaseStore trait (#965) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- query/src/lib.rs | 4 ++-- query/src/test.rs | 4 ++-- server/src/lib.rs | 22 +++++++++++----------- src/influxdb_ioxd/http.rs | 8 ++------ src/influxdb_ioxd/rpc/flight.rs | 1 - src/influxdb_ioxd/rpc/management.rs | 4 ++-- src/influxdb_ioxd/rpc/storage/service.rs | 19 ++++--------------- 7 files changed, 23 insertions(+), 39 deletions(-) diff --git a/query/src/lib.rs b/query/src/lib.rs index a10eaef99f..c3b9fecef6 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -155,11 +155,11 @@ pub trait DatabaseStore: Debug + Send + Sync { type Error: std::error::Error + Send + Sync + 'static; /// List the database names. - async fn db_names_sorted(&self) -> Vec; + fn db_names_sorted(&self) -> Vec; /// Retrieve the database specified by `name` returning None if no /// such database exists - async fn db(&self, name: &str) -> Option>; + fn db(&self, name: &str) -> Option>; /// Retrieve the database specified by `name`, creating it if it /// doesn't exist. diff --git a/query/src/test.rs b/query/src/test.rs index 7fd877f24e..642c5a0f27 100644 --- a/query/src/test.rs +++ b/query/src/test.rs @@ -474,14 +474,14 @@ impl DatabaseStore for TestDatabaseStore { type Error = TestError; /// List the database names. - async fn db_names_sorted(&self) -> Vec { + fn db_names_sorted(&self) -> Vec { let databases = self.databases.lock(); databases.keys().cloned().collect() } /// Retrieve the database specified name - async fn db(&self, name: &str) -> Option> { + fn db(&self, name: &str) -> Option> { let databases = self.databases.lock(); databases.get(name).cloned() diff --git a/server/src/lib.rs b/server/src/lib.rs index e271fdd960..495e28ebbf 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -363,11 +363,11 @@ impl Server { Ok(()) } - pub async fn db(&self, name: &DatabaseName<'_>) -> Option> { + pub fn db(&self, name: &DatabaseName<'_>) -> Option> { self.config.db(name) } - pub async fn db_rules(&self, name: &DatabaseName<'_>) -> Option { + pub fn db_rules(&self, name: &DatabaseName<'_>) -> Option { self.config.db(name).map(|d| d.rules.clone()) } @@ -392,7 +392,7 @@ where type Database = Db; type Error = Error; - async fn db_names_sorted(&self) -> Vec { + fn db_names_sorted(&self) -> Vec { self.config .db_names_sorted() .iter() @@ -400,9 +400,9 @@ where .collect() } - async fn db(&self, name: &str) -> Option> { + fn db(&self, name: &str) -> Option> { if let Ok(name) = DatabaseName::new(name) { - return self.db(&name).await; + return self.db(&name); } None @@ -413,11 +413,11 @@ where async fn db_or_create(&self, name: &str) -> Result, Self::Error> { let db_name = DatabaseName::new(name.to_string()).context(InvalidDatabaseName)?; - let db = match self.db(&db_name).await { + let db = match self.db(&db_name) { Some(db) => db, None => { self.create_database(name, DatabaseRules::new()).await?; - self.db(&db_name).await.expect("db not inserted") + self.db(&db_name).expect("db not inserted") } }; @@ -598,8 +598,8 @@ mod tests { server2.set_id(1); server2.load_database_configs().await.unwrap(); - let _ = server2.db(&DatabaseName::new(db2).unwrap()).await.unwrap(); - let _ = server2.db(&DatabaseName::new(name).unwrap()).await.unwrap(); + let _ = server2.db(&DatabaseName::new(db2).unwrap()).unwrap(); + let _ = server2.db(&DatabaseName::new(name).unwrap()).unwrap(); } #[tokio::test] @@ -648,7 +648,7 @@ mod tests { .expect("failed to create database"); } - let db_names_sorted = server.db_names_sorted().await; + let db_names_sorted = server.db_names_sorted(); assert_eq!(names, db_names_sorted); Ok(()) @@ -693,7 +693,7 @@ mod tests { server.write_lines("foo", &lines).await.unwrap(); let db_name = DatabaseName::new("foo").unwrap(); - let db = server.db(&db_name).await.unwrap(); + let db = server.db(&db_name).unwrap(); let planner = SQLQueryPlanner::default(); let executor = server.executor(); diff --git a/src/influxdb_ioxd/http.rs b/src/influxdb_ioxd/http.rs index 10edba3523..4b47a92608 100644 --- a/src/influxdb_ioxd/http.rs +++ b/src/influxdb_ioxd/http.rs @@ -493,7 +493,6 @@ async fn query( let db = server .db(&db_name) - .await .context(DatabaseNotFound { name: &db_name_str })?; let planner = SQLQueryPlanner::default(); @@ -551,7 +550,6 @@ async fn get_wal_meta( let db = server .db(&db_name) - .await .context(DatabaseNotFound { name: &db_name_str })?; let wal = db @@ -609,7 +607,7 @@ async fn list_partitions( let db_name = org_and_bucket_to_database(&info.org, &info.bucket).context(BucketMappingError)?; - let db = server.db(&db_name).await.context(BucketNotFound { + let db = server.db(&db_name).context(BucketNotFound { org: &info.org, bucket: &info.bucket, })?; @@ -653,7 +651,7 @@ async fn snapshot_partition, ) -> Result, Status> { - let names = self.server.db_names_sorted().await; + let names = self.server.db_names_sorted(); Ok(Response::new(ListDatabasesResponse { names })) } @@ -53,7 +53,7 @@ where ) -> Result, Status> { let name = DatabaseName::new(request.into_inner().name).field("name")?; - match self.server.db_rules(&name).await { + match self.server.db_rules(&name) { Some(rules) => Ok(Response::new(GetDatabaseResponse { rules: Some(rules.into()), })), diff --git a/src/influxdb_ioxd/rpc/storage/service.rs b/src/influxdb_ioxd/rpc/storage/service.rs index 2b026b277c..a2ec41425a 100644 --- a/src/influxdb_ioxd/rpc/storage/service.rs +++ b/src/influxdb_ioxd/rpc/storage/service.rs @@ -757,7 +757,6 @@ where let db = db_store .db(&db_name) - .await .context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); @@ -807,7 +806,7 @@ where })? .build(); - let db = db_store.db(&db_name).await.context(DatabaseNotFound { + let db = db_store.db(&db_name).context(DatabaseNotFound { db_name: db_name.as_str(), })?; @@ -867,10 +866,7 @@ where let db_name = db_name.as_str(); let tag_name = &tag_name; - let db = db_store - .db(db_name) - .await - .context(DatabaseNotFound { db_name })?; + let db = db_store.db(db_name).context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); @@ -927,10 +923,7 @@ where let owned_db_name = db_name; let db_name = owned_db_name.as_str(); - let db = db_store - .db(db_name) - .await - .context(DatabaseNotFound { db_name })?; + let db = db_store.db(db_name).context(DatabaseNotFound { db_name })?; let executor = db_store.executor(); @@ -1018,7 +1011,6 @@ where let db = db_store .db(&db_name) - .await .context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); @@ -1090,10 +1082,7 @@ where .build(); let db_name = db_name.as_str(); - let db = db_store - .db(db_name) - .await - .context(DatabaseNotFound { db_name })?; + let db = db_store.db(db_name).context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); From 8029aa887d155ae0e618e330aae0a16756f70b85 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Thu, 11 Mar 2021 13:54:38 +0100 Subject: [PATCH 027/104] feat: Add subcommands to server --- Dockerfile | 3 +-- docker/Dockerfile.iox | 2 +- src/commands/logging.rs | 4 ++-- src/commands/server.rs | 24 ++++++++++++++++-------- src/influxdb_ioxd.rs | 30 +++++++++++++++--------------- src/main.rs | 25 +++++++++---------------- tests/common/server_fixture.rs | 2 ++ 7 files changed, 46 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index 70b97a2dcf..84535ce2d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,5 +37,4 @@ EXPOSE 8080 8082 ENTRYPOINT ["/usr/bin/influxdb_iox"] -CMD ["server"] - +CMD ["server", "run"] diff --git a/docker/Dockerfile.iox b/docker/Dockerfile.iox index 8b1f33c2a2..f75d76ec19 100644 --- a/docker/Dockerfile.iox +++ b/docker/Dockerfile.iox @@ -21,4 +21,4 @@ EXPOSE 8080 8082 ENTRYPOINT ["influxdb_iox"] -CMD ["server"] +CMD ["server", "run"] diff --git a/src/commands/logging.rs b/src/commands/logging.rs index cb3def640a..bc0386431f 100644 --- a/src/commands/logging.rs +++ b/src/commands/logging.rs @@ -2,7 +2,7 @@ use tracing_subscriber::{prelude::*, EnvFilter}; -use super::server::{Config, LogFormat}; +use super::server::{LogFormat, RunConfig}; /// Handles setting up logging levels #[derive(Debug)] @@ -81,7 +81,7 @@ impl LoggingLevel { /// Configures logging and tracing, based on the configuration /// values, for the IOx server (the whole enchalada) - pub fn setup_logging(&self, config: &Config) -> Option { + pub fn setup_logging(&self, config: &RunConfig) -> Option { // Copy anything from the config to the rust log environment self.set_rust_log_if_needed(config.rust_log.clone()); diff --git a/src/commands/server.rs b/src/commands/server.rs index 9afaa13ed0..de8afcf86a 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -20,15 +20,21 @@ pub const FALLBACK_AWS_REGION: &str = "us-east-1"; #[derive(Debug, Error)] pub enum Error { - #[error("Server error")] + #[error("Server error: {0}")] ServerError(#[from] influxdb_ioxd::Error), } pub type Result = std::result::Result; +#[derive(Debug, StructOpt)] +#[structopt(name = "server", about = "IOx server commands")] +pub enum Config { + Run(RunConfig), +} + #[derive(Debug, StructOpt)] #[structopt( - name = "server", + name = "run", about = "Runs in server mode", long_about = "Run the IOx server.\n\nThe configuration options below can be \ set either with the command line flags or with the specified environment \ @@ -41,7 +47,7 @@ Configuration is loaded from the following sources (highest precedence first): - .env file contents - pre-configured default values" )] -pub struct Config { +pub struct RunConfig { /// This controls the IOx server logging level, as described in /// https://crates.io/crates/env_logger. /// @@ -225,8 +231,10 @@ Possible values (case insensitive): pub jaeger_host: Option, } -pub async fn command(logging_level: LoggingLevel, config: Box) -> Result<()> { - Ok(influxdb_ioxd::main(logging_level, config).await?) +pub async fn command(logging_level: LoggingLevel, config: Config) -> Result<()> { + match config { + Config::Run(config) => Ok(influxdb_ioxd::main(logging_level, config).await?), + } } fn parse_socket_addr(s: &str) -> std::io::Result { @@ -305,7 +313,7 @@ mod tests { #[test] fn test_socketaddr() -> Result<(), clap::Error> { - let c = Config::from_iter_safe( + let c = RunConfig::from_iter_safe( to_vec(&["server", "--api-bind", "127.0.0.1:1234"]).into_iter(), )?; assert_eq!( @@ -313,7 +321,7 @@ mod tests { SocketAddr::from(([127, 0, 0, 1], 1234)) ); - let c = Config::from_iter_safe( + let c = RunConfig::from_iter_safe( to_vec(&["server", "--api-bind", "localhost:1234"]).into_iter(), )?; // depending on where the test runs, localhost will either resolve to a ipv4 or @@ -329,7 +337,7 @@ mod tests { }; assert_eq!( - Config::from_iter_safe( + RunConfig::from_iter_safe( to_vec(&["server", "--api-bind", "!@INv_a1d(ad0/resp_!"]).into_iter(), ) .map_err(|e| e.kind) diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index 5ce1cb4c6c..45b0c57bb9 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -1,6 +1,6 @@ use crate::commands::{ logging::LoggingLevel, - server::{Config, ObjectStore as ObjStoreOpt}, + server::{ObjectStore as ObjStoreOpt, RunConfig}, }; use hyper::Server; use object_store::{ @@ -73,7 +73,7 @@ pub type Result = std::result::Result; /// /// The logging_level passed in is the global setting (e.g. if -v or /// -vv was passed in before 'server') -pub async fn main(logging_level: LoggingLevel, config: Box) -> Result<()> { +pub async fn main(logging_level: LoggingLevel, config: RunConfig) -> Result<()> { // Handle the case if -v/-vv is specified both before and after the server // command let logging_level = logging_level.combine(LoggingLevel::new(config.verbose_count)); @@ -98,7 +98,7 @@ pub async fn main(logging_level: LoggingLevel, config: Box) -> Result<() } } - let object_store = ObjectStore::try_from(&*config)?; + let object_store = ObjectStore::try_from(&config)?; let object_storage = Arc::new(object_store); let connection_manager = ConnectionManager {}; @@ -153,10 +153,10 @@ pub async fn main(logging_level: LoggingLevel, config: Box) -> Result<() Ok(()) } -impl TryFrom<&Config> for ObjectStore { +impl TryFrom<&RunConfig> for ObjectStore { type Error = Error; - fn try_from(config: &Config) -> Result { + fn try_from(config: &RunConfig) -> Result { match config.object_store { Some(ObjStoreOpt::Memory) | None => { Ok(Self::new_in_memory(object_store::memory::InMemory::new())) @@ -283,7 +283,7 @@ mod tests { #[test] fn default_object_store_is_memory() { - let config = Config::from_iter_safe(&["server"]).unwrap(); + let config = RunConfig::from_iter_safe(&["server"]).unwrap(); let object_store = ObjectStore::try_from(&config).unwrap(); @@ -295,7 +295,7 @@ mod tests { #[test] fn explicitly_set_object_store_to_memory() { - let config = Config::from_iter_safe(&["server", "--object-store", "memory"]).unwrap(); + let config = RunConfig::from_iter_safe(&["server", "--object-store", "memory"]).unwrap(); let object_store = ObjectStore::try_from(&config).unwrap(); @@ -307,7 +307,7 @@ mod tests { #[test] fn valid_s3_config() { - let config = Config::from_iter_safe(&[ + let config = RunConfig::from_iter_safe(&[ "server", "--object-store", "s3", @@ -330,7 +330,7 @@ mod tests { #[test] fn s3_config_missing_params() { - let config = Config::from_iter_safe(&["server", "--object-store", "s3"]).unwrap(); + let config = RunConfig::from_iter_safe(&["server", "--object-store", "s3"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); @@ -343,7 +343,7 @@ mod tests { #[test] fn valid_google_config() { - let config = Config::from_iter_safe(&[ + let config = RunConfig::from_iter_safe(&[ "server", "--object-store", "google", @@ -364,7 +364,7 @@ mod tests { #[test] fn google_config_missing_params() { - let config = Config::from_iter_safe(&["server", "--object-store", "google"]).unwrap(); + let config = RunConfig::from_iter_safe(&["server", "--object-store", "google"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); @@ -377,7 +377,7 @@ mod tests { #[test] fn valid_azure_config() { - let config = Config::from_iter_safe(&[ + let config = RunConfig::from_iter_safe(&[ "server", "--object-store", "azure", @@ -400,7 +400,7 @@ mod tests { #[test] fn azure_config_missing_params() { - let config = Config::from_iter_safe(&["server", "--object-store", "azure"]).unwrap(); + let config = RunConfig::from_iter_safe(&["server", "--object-store", "azure"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); @@ -415,7 +415,7 @@ mod tests { fn valid_file_config() { let root = TempDir::new().unwrap(); - let config = Config::from_iter_safe(&[ + let config = RunConfig::from_iter_safe(&[ "server", "--object-store", "file", @@ -434,7 +434,7 @@ mod tests { #[test] fn file_config_missing_params() { - let config = Config::from_iter_safe(&["server", "--object-store", "file"]).unwrap(); + let config = RunConfig::from_iter_safe(&["server", "--object-store", "file"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); diff --git a/src/main.rs b/src/main.rs index 7fe7c2d06c..0f9d1d601d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -83,11 +83,10 @@ struct Config { num_threads: Option, #[structopt(subcommand)] - command: Option, + command: Command, } #[derive(Debug, StructOpt)] -#[structopt(setting = structopt::clap::AppSettings::SubcommandRequiredElseHelp)] enum Command { /// Convert one storage format to another Convert { @@ -132,13 +131,12 @@ fn main() -> Result<(), std::io::Error> { let tokio_runtime = get_runtime(config.num_threads)?; tokio_runtime.block_on(async move { let host = config.host; - match config.command { - Some(Command::Convert { + Command::Convert { input, output, compression_level, - }) => { + } => { logging_level.setup_basic_logging(); let compression_level = CompressionLevel::from_str(&compression_level).unwrap(); @@ -150,7 +148,7 @@ fn main() -> Result<(), std::io::Error> { } } } - Some(Command::Meta { input }) => { + Command::Meta { input } => { logging_level.setup_basic_logging(); match commands::meta::dump_meta(&input) { Ok(()) => debug!("Metadata dump completed successfully"), @@ -160,7 +158,7 @@ fn main() -> Result<(), std::io::Error> { } } } - Some(Command::Stats(config)) => { + Command::Stats(config) => { logging_level.setup_basic_logging(); match commands::stats::stats(&config).await { Ok(()) => debug!("Storage statistics dump completed successfully"), @@ -170,33 +168,28 @@ fn main() -> Result<(), std::io::Error> { } } } - Some(Command::Database(config)) => { + Command::Database(config) => { logging_level.setup_basic_logging(); if let Err(e) = commands::database::command(host, config).await { eprintln!("{}", e); std::process::exit(ReturnCode::Failure as _) } } - Some(Command::Writer(config)) => { + Command::Writer(config) => { logging_level.setup_basic_logging(); if let Err(e) = commands::writer::command(host, config).await { eprintln!("{}", e); std::process::exit(ReturnCode::Failure as _) } } - Some(Command::Server(config)) => { + Command::Server(config) => { // Note don't set up basic logging here, different logging rules apply in server // mode - if let Err(e) = commands::server::command(logging_level, config).await { + if let Err(e) = commands::server::command(logging_level, *config).await { eprintln!("Server command failed: {}", e); std::process::exit(ReturnCode::Failure as _) } } - None => { - unreachable!( - "SubcommandRequiredElseHelp will print help if there is no subcommand" - ); - } } }); diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 550fe41cd4..0b73a2bda2 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -226,6 +226,7 @@ impl TestServer { let server_process = Command::cargo_bin("influxdb_iox") .unwrap() .arg("server") + .arg("run") // Can enable for debugging //.arg("-vv") .env("INFLUXDB_IOX_BIND_ADDR", &addrs.http_bind_addr) @@ -251,6 +252,7 @@ impl TestServer { self.server_process = Command::cargo_bin("influxdb_iox") .unwrap() .arg("server") + .arg("run") // Can enable for debugging //.arg("-vv") .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) From 0606203b4064dfe1b539516c7043eb85119175b7 Mon Sep 17 00:00:00 2001 From: Paul Dix Date: Thu, 11 Mar 2021 15:18:00 -0500 Subject: [PATCH 028/104] feat: add configuration for routing rules This is a strawman for what routing rules might look like in DatabaseRules. Once there's a chance for discussion, I'd move next to looking at how the Server would split up an incoming write into separate FB blobs to be sent to remote IOx servers. That might change what the API/configuration looks like as that's how it would be used (at least for writes). After that it would make sense to move to adding the proto definitions with conversions and gRPC and CLI CRUD to configure routing rules. --- Cargo.lock | 12 +++++ data_types/Cargo.toml | 2 + data_types/src/database_rules.rs | 78 ++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 50de29dc12..12113e896b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -803,7 +803,9 @@ dependencies = [ "generated_types", "influxdb_line_protocol", "percent-encoding", + "regex", "serde", + "serde_regex", "snafu", "test_helpers", "tracing", @@ -3157,6 +3159,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.6.1" diff --git a/data_types/Cargo.toml b/data_types/Cargo.toml index 80cfdc44bc..d18b0fc2bd 100644 --- a/data_types/Cargo.toml +++ b/data_types/Cargo.toml @@ -15,6 +15,8 @@ percent-encoding = "2.1.0" serde = "1.0" snafu = "0.6" tracing = "0.1" +regex = "1.4" +serde_regex = "1.1" [dev-dependencies] # In alphabetical order criterion = "0.3" diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index 5313564c9d..cf692ce4f1 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -1,6 +1,7 @@ use std::convert::{TryFrom, TryInto}; use chrono::{DateTime, TimeZone, Utc}; +use regex::Regex; use serde::{Deserialize, Serialize}; use snafu::Snafu; @@ -51,6 +52,12 @@ pub struct DatabaseRules { /// in object storage. #[serde(default = "MutableBufferConfig::default_option")] pub mutable_buffer_config: Option, + + /// An optional config to route writes or queries to other IOx servers. + /// This is useful for sharding a database or copying all or part of + /// requests over to other servers (like shadow production, etc). + #[serde(default)] + pub routing_config: Option, } impl DatabaseRules { @@ -118,6 +125,7 @@ impl TryFrom for DatabaseRules { partition_template, wal_buffer_config, mutable_buffer_config, + routing_config: None, }) } } @@ -701,6 +709,76 @@ impl TryFrom for TemplatePart { } } +/// RoutingConfig defines rules for routing write or query requests to other IOx +/// servers. In the case of writes, routing rules can be used to create copies +/// of the data to get sent out to other servers in a best effort manner. Each +/// route will be checked with a copy of the write or query sent out to each +/// match. +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] +pub struct RoutingConfig { + routes: Vec, +} + +/// A specific named route to match against. Routes can be done based on +/// specific matches to send to a collection of servers or based on hashing to +/// send to a single server in a collection. +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] +pub struct Route { + /// The name of the route. + pub name: String, + /// An optional matcher. If there is a match, the route will be evaluated to + /// the given targets, otherwise the hash ring will be evaluated. + pub specific_targets: Option, + /// An optional default hasher which will route to one in a collection of + /// nodes. + pub hash_ring: Option, + /// If set to true the router will ignore any errors sent by the remote + /// targets in this route. That is, the write request will succeed + /// regardless of this route's success. + pub ignore_errors: bool, +} + +/// Maps a matcher with specific targets. If it is a match the row/line or query +/// should be sent to all targets. +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] +pub struct MatcherToTargets { + pub matcher: Matcher, + pub targets: Vec, +} + +/// HashRing is a rule for creating a hash key for a row and mapping that to +/// an individual node on a ring. +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] +pub struct HashRing { + /// include the table name in the hash key + pub table_name: Option, + /// include the values of these columns in the hash key + pub columns: Vec, + /// ring of these nodes + pub nodes: Vec, +} + +/// A matcher is used to match routing rules or subscriptions on a row-by-row +/// (or line) basis. +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Matcher { + /// if provided, match if the table name matches against the regex + #[serde(with = "serde_regex")] + pub table_name_regex: Option, + // paul: what should we use for predicate matching here against a single row/line? + pub predicate: Option, +} + +impl PartialEq for Matcher { + fn eq(&self, other: &Self) -> bool { + // this is kind of janky, but it's only used during tests and should get the job + // done + format!("{:?}{:?}", self.table_name_regex, self.predicate) + == format!("{:?}{:?}", other.table_name_regex, other.predicate) + } +} +impl Eq for Matcher {} + /// `PartitionId` is the object storage identifier for a specific partition. It /// should be a path that can be used against an object store to locate all the /// files and subdirectories for a partition. It takes the form of `/ Date: Thu, 11 Mar 2021 16:23:22 +0100 Subject: [PATCH 029/104] feat: Add server remote [set|remove|list] commands --- Cargo.lock | 1 + Cargo.toml | 2 + src/commands/server.rs | 10 +++- src/commands/server_remote.rs | 76 ++++++++++++++++++++++++ src/main.rs | 3 +- tests/end_to_end_cases/management_cli.rs | 59 ++++++++++++++++++ 6 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 src/commands/server_remote.rs diff --git a/Cargo.lock b/Cargo.lock index 50de29dc12..e1101ea91b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1507,6 +1507,7 @@ dependencies = [ "panic_logging", "parking_lot", "predicates", + "prettytable-rs", "prost", "query", "rand 0.7.3", diff --git a/Cargo.toml b/Cargo.toml index 47a3118d39..efd5b79e1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,8 @@ http = "0.2.0" hyper = "0.14" opentelemetry = { version = "0.12", default-features = false, features = ["trace", "tokio-support"] } opentelemetry-jaeger = { version = "0.11", features = ["tokio"] } +# used by arrow/datafusion anyway +prettytable-rs = "0.8" prost = "0.7" # Forked to upgrade hyper and tokio routerify = { git = "https://github.com/influxdata/routerify", rev = "274e250" } diff --git a/src/commands/server.rs b/src/commands/server.rs index de8afcf86a..d270bf2969 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -1,7 +1,7 @@ //! Implementation of command line option for manipulating and showing server //! config -use crate::commands::logging::LoggingLevel; +use crate::commands::{logging::LoggingLevel, server_remote}; use crate::influxdb_ioxd; use clap::arg_enum; use std::{net::SocketAddr, net::ToSocketAddrs, path::PathBuf}; @@ -20,8 +20,10 @@ pub const FALLBACK_AWS_REGION: &str = "us-east-1"; #[derive(Debug, Error)] pub enum Error { - #[error("Server error: {0}")] + #[error("Run: {0}")] ServerError(#[from] influxdb_ioxd::Error), + #[error("Remote: {0}")] + RemoteError(#[from] server_remote::Error), } pub type Result = std::result::Result; @@ -30,6 +32,7 @@ pub type Result = std::result::Result; #[structopt(name = "server", about = "IOx server commands")] pub enum Config { Run(RunConfig), + Remote(crate::commands::server_remote::Config), } #[derive(Debug, StructOpt)] @@ -231,9 +234,10 @@ Possible values (case insensitive): pub jaeger_host: Option, } -pub async fn command(logging_level: LoggingLevel, config: Config) -> Result<()> { +pub async fn command(logging_level: LoggingLevel, url: String, config: Config) -> Result<()> { match config { Config::Run(config) => Ok(influxdb_ioxd::main(logging_level, config).await?), + Config::Remote(config) => Ok(server_remote::command(url, config).await?), } } diff --git a/src/commands/server_remote.rs b/src/commands/server_remote.rs new file mode 100644 index 0000000000..5a6c4fef1c --- /dev/null +++ b/src/commands/server_remote.rs @@ -0,0 +1,76 @@ +use influxdb_iox_client::{connection::Builder, management}; +use structopt::StructOpt; +use thiserror::Error; + +use prettytable::{format, Cell, Row, Table}; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), + + #[error("Update remote error: {0}")] + UpdateError(#[from] management::UpdateRemoteError), + + #[error("List remote error: {0}")] + ListError(#[from] management::ListRemotesError), +} + +pub type Result = std::result::Result; + +#[derive(Debug, StructOpt)] +#[structopt( + name = "remote", + about = "Manage configuration about other IOx servers" +)] +pub enum Config { + /// Set connection parameters for a remote IOx server. + Set { id: u32, connection_string: String }, + /// Remove a reference to a remote IOx server. + Remove { id: u32 }, + /// List configured remote IOx server. + List, +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + + match config { + Config::Set { + id, + connection_string, + } => { + let mut client = management::Client::new(connection); + client.update_remote(id, connection_string).await?; + } + Config::Remove { id } => { + let mut client = management::Client::new(connection); + client.delete_remote(id).await?; + } + Config::List => { + let mut client = management::Client::new(connection); + + let remotes = client.list_remotes().await?; + if remotes.is_empty() { + println!("no remotes configured"); + } else { + let mut table = Table::new(); + table.set_format(*format::consts::FORMAT_NO_LINESEP_WITH_TITLE); + table.set_titles(Row::new(vec![ + Cell::new("ID"), + Cell::new("Connection string"), + ])); + + for i in remotes { + table.add_row(Row::new(vec![ + Cell::new(&format!("{}", i.id)), + Cell::new(&i.connection_string), + ])); + } + print!("{}", table); + } + } + }; + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 0f9d1d601d..3d7e10b201 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ mod commands { pub mod logging; pub mod meta; pub mod server; + pub mod server_remote; pub mod stats; pub mod writer; } @@ -185,7 +186,7 @@ fn main() -> Result<(), std::io::Error> { Command::Server(config) => { // Note don't set up basic logging here, different logging rules apply in server // mode - if let Err(e) = commands::server::command(logging_level, *config).await { + if let Err(e) = commands::server::command(logging_level, host, *config).await { eprintln!("Server command failed: {}", e); std::process::exit(ReturnCode::Failure as _) } diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index c28c5fe56a..81532fbf3b 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -10,6 +10,7 @@ pub async fn test() { test_writer_id(addr).await; test_create_database(addr).await; + test_remotes(addr).await; } async fn test_writer_id(addr: &str) { @@ -81,3 +82,61 @@ async fn test_create_database(addr: &str) { .success() .stdout(predicate::str::contains(format!("name: \"{}\"", db))); } + +async fn test_remotes(addr: &str) { + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("no remotes configured")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("set") + .arg("1") + .arg("http://1.2.3.4:1234") + .arg("--host") + .arg(addr) + .assert() + .success(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("http://1.2.3.4:1234")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("remove") + .arg("1") + .arg("--host") + .arg(addr) + .assert() + .success(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("no remotes configured")); +} From 6ac7e2c1a77cd5d1ac3e93df01c95efeeb48cdff Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 08:56:14 -0500 Subject: [PATCH 030/104] feat: Add management API and CLI to list chunks (#968) * feat: Add management API and CLI to list chunks * fix: Apply suggestions from code review Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> * fix: add comment to protobuf * fix: fix comment * fix: fmt, fixup merge errors * fix: fascinating type dance with prost generated types * fix: clippy * fix: move command to influxdb_iox database chunk list Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- data_types/src/chunk.rs | 176 ++++++++++++++++++ data_types/src/lib.rs | 1 + generated_types/build.rs | 1 + .../influxdata/iox/management/v1/chunk.proto | 37 ++++ .../iox/management/v1/service.proto | 14 ++ .../influxdata/iox/write/v1/service.proto | 2 +- influxdb_iox_client/src/client/management.rs | 23 +++ influxdb_iox_client/src/client/write.rs | 6 +- mutable_buffer/src/database.rs | 8 + mutable_buffer/src/partition.rs | 5 + query/src/lib.rs | 8 +- query/src/test.rs | 4 + read_buffer/src/lib.rs | 29 +++ server/src/db.rs | 104 ++++++++++- server/src/db/chunk.rs | 83 +++++++-- server/src/snapshot.rs | 2 +- src/commands/database.rs | 12 +- src/commands/database/chunk.rs | 73 ++++++++ src/influxdb_ioxd/rpc/error.rs | 29 ++- src/influxdb_ioxd/rpc/management.rs | 37 +++- src/influxdb_ioxd/rpc/write.rs | 6 +- tests/end_to_end_cases/management_api.rs | 128 +++++++++++-- tests/end_to_end_cases/management_cli.rs | 101 ++++++++-- tests/end_to_end_cases/util.rs | 51 +++++ tests/end_to_end_cases/write_api.rs | 15 +- 25 files changed, 879 insertions(+), 76 deletions(-) create mode 100644 data_types/src/chunk.rs create mode 100644 generated_types/protos/influxdata/iox/management/v1/chunk.proto create mode 100644 src/commands/database/chunk.rs diff --git a/data_types/src/chunk.rs b/data_types/src/chunk.rs new file mode 100644 index 0000000000..b68d618b5c --- /dev/null +++ b/data_types/src/chunk.rs @@ -0,0 +1,176 @@ +//! Module contains a representation of chunk metadata +use std::{convert::TryFrom, sync::Arc}; + +use crate::field_validation::FromField; +use generated_types::{google::FieldViolation, influxdata::iox::management::v1 as management}; +use serde::{Deserialize, Serialize}; + +/// Which storage system is a chunk located in? +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)] +pub enum ChunkStorage { + /// The chunk is still open for new writes, in the Mutable Buffer + OpenMutableBuffer, + + /// The chunk is no longer open for writes, in the Mutable Buffer + ClosedMutableBuffer, + + /// The chunk is in the Read Buffer (where it can not be mutated) + ReadBuffer, + + /// The chunk is stored in Object Storage (where it can not be mutated) + ObjectStore, +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)] +/// Represents metadata about a chunk in a database. +/// A chunk can contain one or more tables. +pub struct ChunkSummary { + /// The partitition key of this chunk + pub partition_key: Arc, + + /// The id of this chunk + pub id: u32, + + /// How is this chunk stored? + pub storage: ChunkStorage, + + /// The total estimated size of this chunk, in bytes + pub estimated_bytes: usize, +} + +/// Conversion code to management API chunk structure +impl From for management::Chunk { + fn from(summary: ChunkSummary) -> Self { + let ChunkSummary { + partition_key, + id, + storage, + estimated_bytes, + } = summary; + + let storage: management::ChunkStorage = storage.into(); + let storage = storage.into(); // convert to i32 + + let estimated_bytes = estimated_bytes as u64; + + let partition_key = match Arc::try_unwrap(partition_key) { + // no one else has a reference so take the string + Ok(partition_key) => partition_key, + // some other refernece exists to this string, so clone it + Err(partition_key) => partition_key.as_ref().clone(), + }; + + Self { + partition_key, + id, + storage, + estimated_bytes, + } + } +} + +impl From for management::ChunkStorage { + fn from(storage: ChunkStorage) -> Self { + match storage { + ChunkStorage::OpenMutableBuffer => Self::OpenMutableBuffer, + ChunkStorage::ClosedMutableBuffer => Self::ClosedMutableBuffer, + ChunkStorage::ReadBuffer => Self::ReadBuffer, + ChunkStorage::ObjectStore => Self::ObjectStore, + } + } +} + +/// Conversion code from management API chunk structure +impl TryFrom for ChunkSummary { + type Error = FieldViolation; + + fn try_from(proto: management::Chunk) -> Result { + // Use prost enum conversion + let storage = proto.storage().scope("storage")?; + + let management::Chunk { + partition_key, + id, + estimated_bytes, + .. + } = proto; + + let estimated_bytes = estimated_bytes as usize; + let partition_key = Arc::new(partition_key); + + Ok(Self { + partition_key, + id, + storage, + estimated_bytes, + }) + } +} + +impl TryFrom for ChunkStorage { + type Error = FieldViolation; + + fn try_from(proto: management::ChunkStorage) -> Result { + match proto { + management::ChunkStorage::OpenMutableBuffer => Ok(Self::OpenMutableBuffer), + management::ChunkStorage::ClosedMutableBuffer => Ok(Self::ClosedMutableBuffer), + management::ChunkStorage::ReadBuffer => Ok(Self::ReadBuffer), + management::ChunkStorage::ObjectStore => Ok(Self::ObjectStore), + management::ChunkStorage::Unspecified => Err(FieldViolation::required("")), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn valid_proto_to_summary() { + let proto = management::Chunk { + partition_key: "foo".to_string(), + id: 42, + estimated_bytes: 1234, + storage: management::ChunkStorage::ObjectStore.into(), + }; + + let summary = ChunkSummary::try_from(proto).expect("conversion successful"); + let expected = ChunkSummary { + partition_key: Arc::new("foo".to_string()), + id: 42, + estimated_bytes: 1234, + storage: ChunkStorage::ObjectStore, + }; + + assert_eq!( + summary, expected, + "Actual:\n\n{:?}\n\nExpected:\n\n{:?}\n\n", + summary, expected + ); + } + + #[test] + fn valid_summary_to_proto() { + let summary = ChunkSummary { + partition_key: Arc::new("foo".to_string()), + id: 42, + estimated_bytes: 1234, + storage: ChunkStorage::ObjectStore, + }; + + let proto = management::Chunk::try_from(summary).expect("conversion successful"); + + let expected = management::Chunk { + partition_key: "foo".to_string(), + id: 42, + estimated_bytes: 1234, + storage: management::ChunkStorage::ObjectStore.into(), + }; + + assert_eq!( + proto, expected, + "Actual:\n\n{:?}\n\nExpected:\n\n{:?}\n\n", + proto, expected + ); + } +} diff --git a/data_types/src/lib.rs b/data_types/src/lib.rs index 9b294abaab..c7edc30360 100644 --- a/data_types/src/lib.rs +++ b/data_types/src/lib.rs @@ -20,6 +20,7 @@ pub const TABLE_NAMES_COLUMN_NAME: &str = "table"; /// `column_names`. pub const COLUMN_NAMES_COLUMN_NAME: &str = "column"; +pub mod chunk; pub mod data; pub mod database_rules; pub mod error; diff --git a/generated_types/build.rs b/generated_types/build.rs index 74573e028d..7e66a0f15f 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -39,6 +39,7 @@ fn generate_grpc_types(root: &Path) -> Result<()> { idpe_path.join("source.proto"), management_path.join("base_types.proto"), management_path.join("database_rules.proto"), + management_path.join("chunk.proto"), management_path.join("service.proto"), write_path.join("service.proto"), root.join("grpc/health/v1/service.proto"), diff --git a/generated_types/protos/influxdata/iox/management/v1/chunk.proto b/generated_types/protos/influxdata/iox/management/v1/chunk.proto new file mode 100644 index 0000000000..23b30317b2 --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/chunk.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + + + // Which storage system is a chunk located in? +enum ChunkStorage { + // Not currently returned + CHUNK_STORAGE_UNSPECIFIED = 0; + + // The chunk is still open for new writes, in the Mutable Buffer + CHUNK_STORAGE_OPEN_MUTABLE_BUFFER = 1; + + // The chunk is no longer open for writes, in the Mutable Buffer + CHUNK_STORAGE_CLOSED_MUTABLE_BUFFER = 2; + + // The chunk is in the Read Buffer (where it can not be mutated) + CHUNK_STORAGE_READ_BUFFER = 3; + + // The chunk is stored in Object Storage (where it can not be mutated) + CHUNK_STORAGE_OBJECT_STORE = 4; +} + +// `Chunk` represents part of a partition of data in a database. +// A chunk can contain one or more tables. +message Chunk { + // The partitition key of this chunk + string partition_key = 1; + + // The id of this chunk + uint32 id = 2; + + // Which storage system the chunk is located in + ChunkStorage storage = 3; + + // The total estimated size of this chunk, in bytes + uint64 estimated_bytes = 4; +} diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index f8fb6edd1e..e09f9d652f 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -3,6 +3,7 @@ package influxdata.iox.management.v1; import "google/protobuf/empty.proto"; import "influxdata/iox/management/v1/database_rules.proto"; +import "influxdata/iox/management/v1/chunk.proto"; service ManagementService { rpc GetWriterId(GetWriterIdRequest) returns (GetWriterIdResponse); @@ -15,6 +16,9 @@ service ManagementService { rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + // List chunks available on this database + rpc ListChunks(ListChunksRequest) returns (ListChunksResponse); + // List remote IOx servers we know about. rpc ListRemotes(ListRemotesRequest) returns (ListRemotesResponse); @@ -57,6 +61,16 @@ message CreateDatabaseRequest { message CreateDatabaseResponse {} +message ListChunksRequest { + // the name of the database + string db_name = 1; +} + +message ListChunksResponse { + repeated Chunk chunks = 1; +} + + message ListRemotesRequest {} message ListRemotesResponse { diff --git a/generated_types/protos/influxdata/iox/write/v1/service.proto b/generated_types/protos/influxdata/iox/write/v1/service.proto index 9476204780..c0fa0a9cbf 100644 --- a/generated_types/protos/influxdata/iox/write/v1/service.proto +++ b/generated_types/protos/influxdata/iox/write/v1/service.proto @@ -9,7 +9,7 @@ service WriteService { message WriteRequest { // name of database into which to write - string name = 1; + string db_name = 1; // data, in [LineProtocol] format // diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index df98aaf891..10b0c5bd6a 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -80,6 +80,14 @@ pub enum GetDatabaseError { ServerError(tonic::Status), } +/// Errors returned by Client::list_chunks +#[derive(Debug, Error)] +pub enum ListChunksError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// Errors returned by Client::list_remotes #[derive(Debug, Error)] pub enum ListRemotesError { @@ -215,6 +223,21 @@ impl Client { Ok(rules) } + /// List chunks in a database. + pub async fn list_chunks( + &mut self, + db_name: impl Into, + ) -> Result, ListChunksError> { + let db_name = db_name.into(); + + let response = self + .inner + .list_chunks(ListChunksRequest { db_name }) + .await + .map_err(ListChunksError::ServerError)?; + Ok(response.into_inner().chunks) + } + /// List remotes. pub async fn list_remotes(&mut self) -> Result, ListRemotesError> { let response = self diff --git a/influxdb_iox_client/src/client/write.rs b/influxdb_iox_client/src/client/write.rs index cea5ad12a8..4f57e0bd06 100644 --- a/influxdb_iox_client/src/client/write.rs +++ b/influxdb_iox_client/src/client/write.rs @@ -61,14 +61,14 @@ impl Client { /// [LineProtocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format) pub async fn write( &mut self, - name: impl Into, + db_name: impl Into, lp_data: impl Into, ) -> Result { - let name = name.into(); + let db_name = db_name.into(); let lp_data = lp_data.into(); let response = self .inner - .write(WriteRequest { name, lp_data }) + .write(WriteRequest { db_name, lp_data }) .await .map_err(WriteError::ServerError)?; diff --git a/mutable_buffer/src/database.rs b/mutable_buffer/src/database.rs index 11e0067ee0..200b64c307 100644 --- a/mutable_buffer/src/database.rs +++ b/mutable_buffer/src/database.rs @@ -79,6 +79,14 @@ impl MutableBufferDb { } } + /// returns the id of the current open chunk in the specified partition + pub fn open_chunk_id(&self, partition_key: &str) -> u32 { + let partition = self.get_partition(partition_key); + let partition = partition.read().expect("mutex poisoned"); + + partition.open_chunk_id() + } + /// Directs the writes from batch into the appropriate partitions fn write_entries_to_partitions(&self, batch: &wal::WriteBufferBatch<'_>) -> Result<()> { if let Some(entries) = batch.entries() { diff --git a/mutable_buffer/src/partition.rs b/mutable_buffer/src/partition.rs index e64036056a..0d588cadd0 100644 --- a/mutable_buffer/src/partition.rs +++ b/mutable_buffer/src/partition.rs @@ -107,6 +107,11 @@ impl Partition { } } + /// returns the id of the current open chunk in this partition + pub(crate) fn open_chunk_id(&self) -> u32 { + self.open_chunk.id() + } + /// write data to the open chunk pub fn write_entry(&mut self, entry: &wb::WriteBufferEntry<'_>) -> Result<()> { assert_eq!( diff --git a/query/src/lib.rs b/query/src/lib.rs index c3b9fecef6..b1a3a53b74 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -9,7 +9,8 @@ use arrow_deps::datafusion::physical_plan::SendableRecordBatchStream; use async_trait::async_trait; use data_types::{ - data::ReplicatedWrite, partition_metadata::TableSummary, schema::Schema, selection::Selection, + chunk::ChunkSummary, data::ReplicatedWrite, partition_metadata::TableSummary, schema::Schema, + selection::Selection, }; use exec::{stringset::StringSet, Executor}; @@ -49,6 +50,9 @@ pub trait Database: Debug + Send + Sync { /// covering set means that together the chunks make up a single /// complete copy of the data being queried. fn chunks(&self, partition_key: &str) -> Vec>; + + /// Return a summary of all chunks in this database, in all partitions + fn chunk_summaries(&self) -> Result, Self::Error>; } /// Collection of data that shares the same partition key @@ -60,7 +64,7 @@ pub trait PartitionChunk: Debug + Send + Sync { /// particular partition. fn id(&self) -> u32; - /// returns the partition metadata stats for every table in the partition + /// returns the partition metadata stats for every table in the chunk fn table_stats(&self) -> Result, Self::Error>; /// Returns true if this chunk *might* have data that passes the diff --git a/query/src/test.rs b/query/src/test.rs index 642c5a0f27..8117dae1df 100644 --- a/query/src/test.rs +++ b/query/src/test.rs @@ -154,6 +154,10 @@ impl Database for TestDatabase { vec![] } } + + fn chunk_summaries(&self) -> Result, Self::Error> { + unimplemented!("summaries not implemented TestDatabase") + } } #[derive(Debug, Default)] diff --git a/read_buffer/src/lib.rs b/read_buffer/src/lib.rs index b8d9f13751..b0df18b8e5 100644 --- a/read_buffer/src/lib.rs +++ b/read_buffer/src/lib.rs @@ -177,6 +177,25 @@ impl Database { .unwrap_or_default() } + /// Returns the total estimated size in bytes for the chunks in the + /// specified partition. Returns None if there is no such partition + pub fn chunks_size<'a>( + &self, + partition_key: &str, + chunk_ids: impl IntoIterator, + ) -> Option { + let partition_data = self.data.read().unwrap(); + + let partition = partition_data.partitions.get(partition_key); + + partition.map(|partition| { + chunk_ids + .into_iter() + .map(|chunk_id| partition.chunk_size(*chunk_id)) + .sum::() + }) + } + /// Returns the total estimated size in bytes of the database. pub fn size(&self) -> u64 { let base_size = std::mem::size_of::(); @@ -662,6 +681,16 @@ impl Partition { .map(|chunk| std::mem::size_of::() as u64 + chunk.size()) .sum::() } + + /// The total estimated size in bytes of the specified chunk id + pub fn chunk_size(&self, chunk_id: u32) -> u64 { + let chunk_data = self.data.read().unwrap(); + chunk_data + .chunks + .get(&chunk_id) + .map(|chunk| chunk.size()) + .unwrap_or(0) // treat unknown chunks as zero size + } } /// ReadFilterResults implements ... diff --git a/server/src/db.rs b/server/src/db.rs index b5ad60244b..e026994162 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -10,7 +10,9 @@ use std::{ }; use async_trait::async_trait; -use data_types::{data::ReplicatedWrite, database_rules::DatabaseRules, selection::Selection}; +use data_types::{ + chunk::ChunkSummary, data::ReplicatedWrite, database_rules::DatabaseRules, selection::Selection, +}; use mutable_buffer::MutableBufferDb; use parking_lot::Mutex; use query::{Database, PartitionChunk}; @@ -121,7 +123,7 @@ impl Db { local_store .rollover_partition(partition_key) .context(RollingPartition) - .map(DBChunk::new_mb) + .map(|c| DBChunk::new_mb(c, partition_key, false)) } else { DatatbaseNotWriteable {}.fail() } @@ -131,10 +133,15 @@ impl Db { // potentially be migrated into the read buffer or object store) pub fn mutable_buffer_chunks(&self, partition_key: &str) -> Vec> { let chunks = if let Some(mutable_buffer) = self.mutable_buffer.as_ref() { + let open_chunk_id = mutable_buffer.open_chunk_id(partition_key); + mutable_buffer .chunks(partition_key) .into_iter() - .map(DBChunk::new_mb) + .map(|c| { + let open = c.id() == open_chunk_id; + DBChunk::new_mb(c, partition_key, open) + }) .collect() } else { vec![] @@ -162,7 +169,7 @@ impl Db { .as_ref() .context(DatatbaseNotWriteable)? .drop_chunk(partition_key, chunk_id) - .map(DBChunk::new_mb) + .map(|c| DBChunk::new_mb(c, partition_key, false)) .context(MutableBufferDrop) } @@ -313,6 +320,21 @@ impl Database for Db { .partition_keys() .context(MutableBufferRead) } + + fn chunk_summaries(&self) -> Result> { + let summaries = self + .partition_keys()? + .into_iter() + .map(|partition_key| { + self.mutable_buffer_chunks(&partition_key) + .into_iter() + .chain(self.read_buffer_chunks(&partition_key).into_iter()) + .map(|c| c.summary()) + }) + .flatten() + .collect(); + Ok(summaries) + } } #[cfg(test)] @@ -324,8 +346,9 @@ mod tests { use arrow_deps::{ arrow::record_batch::RecordBatch, assert_table_eq, datafusion::physical_plan::collect, }; - use data_types::database_rules::{ - MutableBufferConfig, Order, PartitionSort, PartitionSortRules, + use data_types::{ + chunk::ChunkStorage, + database_rules::{MutableBufferConfig, Order, PartitionSort, PartitionSortRules}, }; use query::{ exec::Executor, frontend::sql::SQLQueryPlanner, test::TestLPWriter, PartitionChunk, @@ -559,6 +582,75 @@ mod tests { db.rules.mutable_buffer_config = Some(mbconf); } + #[tokio::test] + async fn chunk_summaries() { + // Test that chunk id listing is hooked up + let db = make_db(); + let mut writer = TestLPWriter::default(); + + // get three chunks: one open, one closed in mb and one close in rb + + writer.write_lp_string(&db, "cpu bar=1 1").await.unwrap(); + db.rollover_partition("1970-01-01T00").await.unwrap(); + + writer + .write_lp_string(&db, "cpu bar=1,baz=2 2") + .await + .unwrap(); + + // a fourth chunk in a different partition + writer + .write_lp_string(&db, "cpu bar=1,baz2,frob=3 400000000000000") + .await + .unwrap(); + + print!("Partitions: {:?}", db.partition_keys().unwrap()); + + db.load_chunk_to_read_buffer("1970-01-01T00", 0) + .await + .unwrap(); + + fn to_arc(s: &str) -> Arc { + Arc::new(s.to_string()) + } + + let mut chunk_summaries = db.chunk_summaries().expect("expected summary to return"); + chunk_summaries.sort_unstable(); + + let expected = vec![ + ChunkSummary { + partition_key: to_arc("1970-01-01T00"), + id: 0, + storage: ChunkStorage::ClosedMutableBuffer, + estimated_bytes: 70, + }, + ChunkSummary { + partition_key: to_arc("1970-01-01T00"), + id: 0, + storage: ChunkStorage::ReadBuffer, + estimated_bytes: 1221, + }, + ChunkSummary { + partition_key: to_arc("1970-01-01T00"), + id: 1, + storage: ChunkStorage::OpenMutableBuffer, + estimated_bytes: 101, + }, + ChunkSummary { + partition_key: to_arc("1970-01-05T15"), + id: 0, + storage: ChunkStorage::OpenMutableBuffer, + estimated_bytes: 107, + }, + ]; + + assert_eq!( + expected, chunk_summaries, + "expected:\n{:#?}\n\nactual:{:#?}\n\n", + expected, chunk_summaries + ); + } + // run a sql query against the database, returning the results as record batches async fn run_query(db: &Db, query: &str) -> Vec { let planner = SQLQueryPlanner::default(); diff --git a/server/src/db/chunk.rs b/server/src/db/chunk.rs index 185f033d32..377edf546f 100644 --- a/server/src/db/chunk.rs +++ b/server/src/db/chunk.rs @@ -1,5 +1,9 @@ use arrow_deps::datafusion::physical_plan::SendableRecordBatchStream; -use data_types::{schema::Schema, selection::Selection}; +use data_types::{ + chunk::{ChunkStorage, ChunkSummary}, + schema::Schema, + selection::Selection, +}; use mutable_buffer::chunk::Chunk as MBChunk; use query::{exec::stringset::StringSet, predicate::Predicate, PartitionChunk}; use read_buffer::Database as ReadBufferDb; @@ -58,10 +62,13 @@ pub type Result = std::result::Result; pub enum DBChunk { MutableBuffer { chunk: Arc, + partition_key: Arc, + /// is this chunk open for writing? + open: bool, }, ReadBuffer { db: Arc, - partition_key: String, + partition_key: Arc, chunk_id: u32, }, ParquetFile, // TODO add appropriate type here @@ -69,8 +76,17 @@ pub enum DBChunk { impl DBChunk { /// Create a new mutable buffer chunk - pub fn new_mb(chunk: Arc) -> Arc { - Arc::new(Self::MutableBuffer { chunk }) + pub fn new_mb( + chunk: Arc, + partition_key: impl Into, + open: bool, + ) -> Arc { + let partition_key = Arc::new(partition_key.into()); + Arc::new(Self::MutableBuffer { + chunk, + partition_key, + open, + }) } /// create a new read buffer chunk @@ -79,13 +95,54 @@ impl DBChunk { partition_key: impl Into, chunk_id: u32, ) -> Arc { - let partition_key = partition_key.into(); + let partition_key = Arc::new(partition_key.into()); Arc::new(Self::ReadBuffer { db, chunk_id, partition_key, }) } + + pub fn summary(&self) -> ChunkSummary { + match self { + Self::MutableBuffer { + chunk, + partition_key, + open, + } => { + let storage = if *open { + ChunkStorage::OpenMutableBuffer + } else { + ChunkStorage::ClosedMutableBuffer + }; + ChunkSummary { + partition_key: Arc::clone(partition_key), + id: chunk.id(), + storage, + estimated_bytes: chunk.size(), + } + } + Self::ReadBuffer { + db, + partition_key, + chunk_id, + } => { + let estimated_bytes = db + .chunks_size(partition_key.as_ref(), &[*chunk_id]) + .unwrap_or(0) as usize; + + ChunkSummary { + partition_key: Arc::clone(&partition_key), + id: *chunk_id, + storage: ChunkStorage::ReadBuffer, + estimated_bytes, + } + } + Self::ParquetFile => { + unimplemented!("parquet file summary not implemented") + } + } + } } #[async_trait] @@ -94,7 +151,7 @@ impl PartitionChunk for DBChunk { fn id(&self) -> u32 { match self { - Self::MutableBuffer { chunk } => chunk.id(), + Self::MutableBuffer { chunk, .. } => chunk.id(), Self::ReadBuffer { chunk_id, .. } => *chunk_id, Self::ParquetFile => unimplemented!("parquet file not implemented"), } @@ -104,7 +161,7 @@ impl PartitionChunk for DBChunk { &self, ) -> Result, Self::Error> { match self { - Self::MutableBuffer { chunk } => chunk.table_stats().context(MutableBufferChunk), + Self::MutableBuffer { chunk, .. } => chunk.table_stats().context(MutableBufferChunk), Self::ReadBuffer { .. } => unimplemented!("read buffer not implemented"), Self::ParquetFile => unimplemented!("parquet file not implemented"), } @@ -116,7 +173,7 @@ impl PartitionChunk for DBChunk { _known_tables: &StringSet, ) -> Result, Self::Error> { let names = match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { if chunk.is_empty() { Some(StringSet::new()) } else { @@ -192,7 +249,7 @@ impl PartitionChunk for DBChunk { selection: Selection<'_>, ) -> Result { match self { - DBChunk::MutableBuffer { chunk } => chunk + DBChunk::MutableBuffer { chunk, .. } => chunk .table_schema(table_name, selection) .context(MutableBufferChunk), DBChunk::ReadBuffer { @@ -235,7 +292,7 @@ impl PartitionChunk for DBChunk { fn has_table(&self, table_name: &str) -> bool { match self { - Self::MutableBuffer { chunk } => chunk.has_table(table_name), + Self::MutableBuffer { chunk, .. } => chunk.has_table(table_name), Self::ReadBuffer { db, partition_key, @@ -257,7 +314,7 @@ impl PartitionChunk for DBChunk { selection: Selection<'_>, ) -> Result { match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { // Note MutableBuffer doesn't support predicate // pushdown (other than pruning out the entire chunk // via `might_pass_predicate) @@ -341,7 +398,7 @@ impl PartitionChunk for DBChunk { columns: Selection<'_>, ) -> Result, Self::Error> { match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { let chunk_predicate = match to_mutable_buffer_predicate(chunk, predicate) { Ok(chunk_predicate) => chunk_predicate, Err(e) => { @@ -389,7 +446,7 @@ impl PartitionChunk for DBChunk { predicate: &Predicate, ) -> Result, Self::Error> { match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { use mutable_buffer::chunk::Error::UnsupportedColumnTypeForListingValues; let chunk_predicate = match to_mutable_buffer_predicate(chunk, predicate) { diff --git a/server/src/snapshot.rs b/server/src/snapshot.rs index 69a7d1726f..403435fdfa 100644 --- a/server/src/snapshot.rs +++ b/server/src/snapshot.rs @@ -442,7 +442,7 @@ mem,host=A,region=west used=45 1 ]; let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); - let chunk = DBChunk::new_mb(Arc::new(ChunkWB::new(11))); + let chunk = DBChunk::new_mb(Arc::new(ChunkWB::new(11)), "key", false); let mut metadata_path = store.new_path(); metadata_path.push_dir("meta"); diff --git a/src/commands/database.rs b/src/commands/database.rs index 5efbc80847..74b2a12df1 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -13,6 +13,8 @@ use influxdb_iox_client::{ use structopt::StructOpt; use thiserror::Error; +mod chunk; + #[derive(Debug, Error)] pub enum Error { #[error("Error creating database: {0}")] @@ -41,6 +43,9 @@ pub enum Error { #[error("Error querying: {0}")] Query(#[from] influxdb_iox_client::flight::Error), + + #[error("Error in chunk subcommand: {0}")] + Chunk(#[from] chunk::Error), } pub type Result = std::result::Result; @@ -101,10 +106,11 @@ enum Command { Get(Get), Write(Write), Query(Query), + Chunk(chunk::Config), } pub async fn command(url: String, config: Config) -> Result<()> { - let connection = Builder::default().build(url).await?; + let connection = Builder::default().build(url.clone()).await?; match config.command { Command::Create(command) => { @@ -173,8 +179,12 @@ pub async fn command(url: String, config: Config) -> Result<()> { } let formatted_result = format.format(&batches)?; + println!("{}", formatted_result); } + Command::Chunk(config) => { + chunk::command(url, config).await?; + } } Ok(()) diff --git a/src/commands/database/chunk.rs b/src/commands/database/chunk.rs new file mode 100644 index 0000000000..6f8c3a3806 --- /dev/null +++ b/src/commands/database/chunk.rs @@ -0,0 +1,73 @@ +//! This module implements the `chunk` CLI command +use data_types::chunk::ChunkSummary; +use generated_types::google::FieldViolation; +use influxdb_iox_client::{ + connection::Builder, + management::{self, ListChunksError}, +}; +use std::convert::TryFrom; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error listing chunks: {0}")] + ListChunkError(#[from] ListChunksError), + + #[error("Error interpreting server response: {:?}", .0)] + ConvertingResponse(FieldViolation), + + #[error("Error rendering response as JSON: {0}")] + WritingJson(#[from] serde_json::Error), + + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), +} + +pub type Result = std::result::Result; + +/// Manage IOx databases +#[derive(Debug, StructOpt)] +pub struct Config { + #[structopt(subcommand)] + command: Command, +} + +/// List the chunks for the specified database in JSON format +#[derive(Debug, StructOpt)] +struct List { + /// The name of the database + db_name: String, +} + +/// All possible subcommands for chunk +#[derive(Debug, StructOpt)] +enum Command { + List(List), +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + + match config.command { + Command::List(get) => { + let List { db_name } = get; + + let mut client = management::Client::new(connection); + + let chunks = client + .list_chunks(db_name) + .await + .map_err(Error::ListChunkError)?; + + let chunks = chunks + .into_iter() + .map(|c| ChunkSummary::try_from(c).map_err(Error::ConvertingResponse)) + .collect::>>()?; + + serde_json::to_writer_pretty(std::io::stdout(), &chunks).map_err(Error::WritingJson)?; + } + } + + Ok(()) +} diff --git a/src/influxdb_ioxd/rpc/error.rs b/src/influxdb_ioxd/rpc/error.rs index 8c80714a01..e6f71d8dc3 100644 --- a/src/influxdb_ioxd/rpc/error.rs +++ b/src/influxdb_ioxd/rpc/error.rs @@ -1,10 +1,10 @@ use generated_types::google::{InternalError, NotFound, PreconditionViolation}; use tracing::error; -use server::Error; +/// map common `server::Error` errors to the appropriate tonic Status +pub fn default_server_error_handler(error: server::Error) -> tonic::Status { + use server::Error; -/// map common server errors to the appropriate tonic Status -pub fn default_error_handler(error: Error) -> tonic::Status { match error { Error::IdNotSet => PreconditionViolation { category: "Writer ID".to_string(), @@ -24,3 +24,26 @@ pub fn default_error_handler(error: Error) -> tonic::Status { } } } + +/// map common `server::db::Error` errors to the appropriate tonic Status +pub fn default_db_error_handler(error: server::db::Error) -> tonic::Status { + use server::db::Error; + match error { + Error::DatabaseNotReadable {} => PreconditionViolation { + category: "database".to_string(), + subject: "influxdata.com/iox".to_string(), + description: "Cannot read from database: no mutable buffer configured".to_string(), + } + .into(), + Error::DatatbaseNotWriteable {} => PreconditionViolation { + category: "database".to_string(), + subject: "influxdata.com/iox".to_string(), + description: "Cannot write to database: no mutable buffer configured".to_string(), + } + .into(), + error => { + error!(?error, "Unexpected error"); + InternalError {}.into() + } + } +} diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index 926588f09c..570d57875a 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -6,7 +6,7 @@ use data_types::database_rules::DatabaseRules; use data_types::DatabaseName; use generated_types::google::{AlreadyExists, FieldViolation, FieldViolationExt, NotFound}; use generated_types::influxdata::iox::management::v1::*; -use query::DatabaseStore; +use query::{Database, DatabaseStore}; use server::{ConnectionManager, Error, Server}; use tonic::{Request, Response, Status}; @@ -14,7 +14,7 @@ struct ManagementService { server: Arc>, } -use super::error::default_error_handler; +use super::error::{default_db_error_handler, default_server_error_handler}; #[tonic::async_trait] impl management_service_server::ManagementService for ManagementService @@ -92,10 +92,41 @@ where } .into()) } - Err(e) => Err(default_error_handler(e)), + Err(e) => Err(default_server_error_handler(e)), } } + async fn list_chunks( + &self, + request: Request, + ) -> Result, Status> { + let db_name = DatabaseName::new(request.into_inner().db_name).field("db_name")?; + + let db = match self.server.db(&db_name) { + Some(db) => db, + None => { + return Err(NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + } + .into()) + } + }; + + let chunk_summaries = match db.chunk_summaries() { + Ok(chunk_summaries) => chunk_summaries, + Err(e) => return Err(default_db_error_handler(e)), + }; + + let chunks: Vec = chunk_summaries + .into_iter() + .map(|summary| summary.into()) + .collect(); + + Ok(Response::new(ListChunksResponse { chunks })) + } + async fn list_remotes( &self, _: Request, diff --git a/src/influxdb_ioxd/rpc/write.rs b/src/influxdb_ioxd/rpc/write.rs index 2b3eafaea1..67e48fb438 100644 --- a/src/influxdb_ioxd/rpc/write.rs +++ b/src/influxdb_ioxd/rpc/write.rs @@ -7,7 +7,7 @@ use std::fmt::Debug; use tonic::Response; use tracing::debug; -use super::error::default_error_handler; +use super::error::default_server_error_handler; /// Implementation of the write service struct WriteService { @@ -25,7 +25,7 @@ where ) -> Result, tonic::Status> { let request = request.into_inner(); - let db_name = request.name; + let db_name = request.db_name; let lp_data = request.lp_data; let lp_chars = lp_data.len(); @@ -42,7 +42,7 @@ where self.server .write_lines(&db_name, &lines) .await - .map_err(default_error_handler)?; + .map_err(default_server_error_handler)?; let lines_written = lp_line_count as u64; Ok(Response::new(WriteResponse { lines_written })) diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index ae21281858..2bc096ace1 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -3,25 +3,17 @@ use std::num::NonZeroU32; use generated_types::google::protobuf::Empty; use generated_types::{google::protobuf::Duration, influxdata::iox::management::v1::*}; use influxdb_iox_client::management::{Client, CreateDatabaseError}; +use test_helpers::assert_contains; use crate::common::server_fixture::ServerFixture; -use super::util::rand_name; +use super::util::{create_readable_database, create_unreadable_database, rand_name}; #[tokio::test] -pub async fn test() { +async fn test_list_update_remotes() { let server_fixture = ServerFixture::create_single_use().await; let mut client = Client::new(server_fixture.grpc_channel()); - test_list_update_remotes(&mut client).await; - test_set_get_writer_id(&mut client).await; - test_create_database_duplicate_name(&mut client).await; - test_create_database_invalid_name(&mut client).await; - test_list_databases(&mut client).await; - test_create_get_database(&mut client).await; -} - -async fn test_list_update_remotes(client: &mut Client) { const TEST_REMOTE_ID_1: u32 = 42; const TEST_REMOTE_ADDR_1: &str = "1.2.3.4:1234"; const TEST_REMOTE_ID_2: u32 = 84; @@ -77,7 +69,11 @@ async fn test_list_update_remotes(client: &mut Client) { assert_eq!(res[0].connection_string, TEST_REMOTE_ADDR_2_UPDATED); } -async fn test_set_get_writer_id(client: &mut Client) { +#[tokio::test] +async fn test_set_get_writer_id() { + let server_fixture = ServerFixture::create_single_use().await; + let mut client = Client::new(server_fixture.grpc_channel()); + const TEST_ID: u32 = 42; client @@ -90,7 +86,11 @@ async fn test_set_get_writer_id(client: &mut Client) { assert_eq!(got.get(), TEST_ID); } -async fn test_create_database_duplicate_name(client: &mut Client) { +#[tokio::test] +async fn test_create_database_duplicate_name() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = Client::new(server_fixture.grpc_channel()); + let db_name = rand_name(); client @@ -115,7 +115,11 @@ async fn test_create_database_duplicate_name(client: &mut Client) { )) } -async fn test_create_database_invalid_name(client: &mut Client) { +#[tokio::test] +async fn test_create_database_invalid_name() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = Client::new(server_fixture.grpc_channel()); + let err = client .create_database(DatabaseRules { name: "my_example\ndb".to_string(), @@ -127,7 +131,11 @@ async fn test_create_database_invalid_name(client: &mut Client) { assert!(matches!(dbg!(err), CreateDatabaseError::InvalidArgument(_))); } -async fn test_list_databases(client: &mut Client) { +#[tokio::test] +async fn test_list_databases() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = Client::new(server_fixture.grpc_channel()); + let name = rand_name(); client .create_database(DatabaseRules { @@ -144,7 +152,11 @@ async fn test_list_databases(client: &mut Client) { assert!(names.contains(&name)); } -async fn test_create_get_database(client: &mut Client) { +#[tokio::test] +async fn test_create_get_database() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = Client::new(server_fixture.grpc_channel()); + let db_name = rand_name(); // Specify everything to allow direct comparison between request and response @@ -191,3 +203,87 @@ async fn test_create_get_database(client: &mut Client) { assert_eq!(response, rules); } + +#[tokio::test] +async fn test_chunk_get() { + use generated_types::influxdata::iox::management::v1::{Chunk, ChunkStorage}; + + let fixture = ServerFixture::create_shared().await; + let mut management_client = Client::new(fixture.grpc_channel()); + let mut write_client = influxdb_iox_client::write::Client::new(fixture.grpc_channel()); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + "disk,region=east bytes=99i 200", + ]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let mut chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + // ensure the output order is consistent + chunks.sort_by(|c1, c2| c1.partition_key.cmp(&c2.partition_key)); + + let expected: Vec = vec![ + Chunk { + partition_key: "cpu".into(), + id: 0, + storage: ChunkStorage::OpenMutableBuffer as i32, + estimated_bytes: 145, + }, + Chunk { + partition_key: "disk".into(), + id: 0, + storage: ChunkStorage::OpenMutableBuffer as i32, + estimated_bytes: 107, + }, + ]; + assert_eq!( + expected, chunks, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, chunks + ); +} + +#[tokio::test] +async fn test_chunk_get_errors() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = Client::new(fixture.grpc_channel()); + let db_name = rand_name(); + + let err = management_client + .list_chunks(&db_name) + .await + .expect_err("no db had been created"); + + assert_contains!( + err.to_string(), + "Some requested entity was not found: Resource database" + ); + + create_unreadable_database(&db_name, fixture.grpc_channel()).await; + + let err = management_client + .list_chunks(&db_name) + .await + .expect_err("db can't be read"); + + assert_contains!( + err.to_string(), + "Precondition violation influxdata.com/iox - database" + ); + assert_contains!( + err.to_string(), + "Cannot read from database: no mutable buffer configured" + ); +} diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 81532fbf3b..8ef8688682 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -1,19 +1,15 @@ use assert_cmd::Command; use predicates::prelude::*; +use test_helpers::make_temp_file; use crate::common::server_fixture::ServerFixture; +use super::util::{create_readable_database, rand_name}; + #[tokio::test] -pub async fn test() { - let server_fixture = ServerFixture::create_single_use().await; +async fn test_writer_id() { + let server_fixture = ServerFixture::create_shared().await; let addr = server_fixture.grpc_base(); - - test_writer_id(addr).await; - test_create_database(addr).await; - test_remotes(addr).await; -} - -async fn test_writer_id(addr: &str) { Command::cargo_bin("influxdb_iox") .unwrap() .arg("writer") @@ -36,8 +32,12 @@ async fn test_writer_id(addr: &str) { .stdout(predicate::str::contains("32")); } -async fn test_create_database(addr: &str) { - let db = "management-cli-test"; +#[tokio::test] +async fn test_create_database() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + let db = &db_name; Command::cargo_bin("influxdb_iox") .unwrap() @@ -83,7 +83,84 @@ async fn test_create_database(addr: &str) { .stdout(predicate::str::contains(format!("name: \"{}\"", db))); } -async fn test_remotes(addr: &str) { +#[tokio::test] +async fn test_get_chunks() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + ]; + + let lp_data_file = make_temp_file(lp_data.join("\n")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(&db_name) + .arg(lp_data_file.as_ref()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("2 Lines OK")); + + let expected = r#"[ + { + "partition_key": "cpu", + "id": 0, + "storage": "OpenMutableBuffer", + "estimated_bytes": 145 + } +]"#; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("chunk") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +#[tokio::test] +async fn test_list_chunks_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + // note don't make the database, expect error + + // list the chunks + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("chunk") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr( + predicate::str::contains("Some requested entity was not found: Resource database") + .and(predicate::str::contains(&db_name)), + ); +} + +#[tokio::test] +async fn test_remotes() { + let server_fixture = ServerFixture::create_single_use().await; + let addr = server_fixture.grpc_base(); Command::cargo_bin("influxdb_iox") .unwrap() .arg("server") diff --git a/tests/end_to_end_cases/util.rs b/tests/end_to_end_cases/util.rs index f70c1cd241..a259fc9081 100644 --- a/tests/end_to_end_cases/util.rs +++ b/tests/end_to_end_cases/util.rs @@ -1,5 +1,8 @@ use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use generated_types::google::protobuf::Empty; +use generated_types::influxdata::iox::management::v1::*; + /// Return a random string suitable for use as a database name pub fn rand_name() -> String { thread_rng() @@ -8,3 +11,51 @@ pub fn rand_name() -> String { .map(char::from) .collect() } + +/// given a channel to talk with the managment api, create a new +/// database with the specified name configured with a 10MB mutable +/// buffer, partitioned on table +pub async fn create_readable_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + let rules = DatabaseRules { + name: db_name.into(), + partition_template: Some(PartitionTemplate { + parts: vec![partition_template::Part { + part: Some(partition_template::part::Part::Table(Empty {})), + }], + }), + mutable_buffer_config: Some(MutableBufferConfig { + buffer_size: 10 * 1024 * 1024, + ..Default::default() + }), + ..Default::default() + }; + + management_client + .create_database(rules.clone()) + .await + .expect("create database failed"); +} + +/// given a channel to talk with the managment api, create a new +/// database with no mutable buffer configured, no partitioning rules +pub async fn create_unreadable_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + let rules = DatabaseRules { + name: db_name.into(), + ..Default::default() + }; + + management_client + .create_database(rules.clone()) + .await + .expect("create database failed"); +} diff --git a/tests/end_to_end_cases/write_api.rs b/tests/end_to_end_cases/write_api.rs index 9785e9e722..a1a67aa283 100644 --- a/tests/end_to_end_cases/write_api.rs +++ b/tests/end_to_end_cases/write_api.rs @@ -1,26 +1,17 @@ -use influxdb_iox_client::management::{self, generated_types::DatabaseRules}; use influxdb_iox_client::write::{self, WriteError}; use test_helpers::assert_contains; -use super::util::rand_name; - use crate::common::server_fixture::ServerFixture; +use super::util::{create_readable_database, rand_name}; + #[tokio::test] async fn test_write() { let fixture = ServerFixture::create_shared().await; - let mut management_client = management::Client::new(fixture.grpc_channel()); let mut write_client = write::Client::new(fixture.grpc_channel()); let db_name = rand_name(); - - management_client - .create_database(DatabaseRules { - name: db_name.clone(), - ..Default::default() - }) - .await - .expect("create database failed"); + create_readable_database(&db_name, fixture.grpc_channel()).await; let lp_lines = vec![ "cpu,region=west user=23.2 100", From 2456bc1b28daa0042187c88c101e8f7917360bfd Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 09:30:45 -0500 Subject: [PATCH 031/104] refactor: impl Error for FieldViolation (#975) --- generated_types/src/google.rs | 14 +++++++++++++- src/commands/database/chunk.rs | 8 ++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/generated_types/src/google.rs b/generated_types/src/google.rs index 51339d87bc..75b204397e 100644 --- a/generated_types/src/google.rs +++ b/generated_types/src/google.rs @@ -84,6 +84,18 @@ impl FieldViolation { } } +impl std::error::Error for FieldViolation {} + +impl std::fmt::Display for FieldViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Violation for field \"{}\": {}", + self.field, self.description + ) + } +} + fn encode_bad_request(violation: Vec) -> Result { let mut buffer = BytesMut::new(); @@ -106,7 +118,7 @@ fn encode_bad_request(violation: Vec) -> Result for tonic::Status { fn from(f: FieldViolation) -> Self { - let message = format!("Violation for field \"{}\": {}", f.field, f.description); + let message = f.to_string(); match encode_bad_request(vec![f]) { Ok(details) => encode_status(tonic::Code::InvalidArgument, message, details), diff --git a/src/commands/database/chunk.rs b/src/commands/database/chunk.rs index 6f8c3a3806..b5ab4cf70c 100644 --- a/src/commands/database/chunk.rs +++ b/src/commands/database/chunk.rs @@ -14,8 +14,8 @@ pub enum Error { #[error("Error listing chunks: {0}")] ListChunkError(#[from] ListChunksError), - #[error("Error interpreting server response: {:?}", .0)] - ConvertingResponse(FieldViolation), + #[error("Error interpreting server response: {0}")] + ConvertingResponse(#[from] FieldViolation), #[error("Error rendering response as JSON: {0}")] WritingJson(#[from] serde_json::Error), @@ -62,8 +62,8 @@ pub async fn command(url: String, config: Config) -> Result<()> { let chunks = chunks .into_iter() - .map(|c| ChunkSummary::try_from(c).map_err(Error::ConvertingResponse)) - .collect::>>()?; + .map(ChunkSummary::try_from) + .collect::, FieldViolation>>()?; serde_json::to_writer_pretty(std::io::stdout(), &chunks).map_err(Error::WritingJson)?; } From 6fecf68bd40adba2d1aed112a8fd0ab81e52f844 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 09:43:49 -0500 Subject: [PATCH 032/104] feat: make CLI to listing database consistent with other commands (#974) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- src/commands/database.rs | 27 ++++++++++++++---------- tests/end_to_end_cases/management_cli.rs | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/commands/database.rs b/src/commands/database.rs index 74b2a12df1..208cc4e879 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -68,11 +68,15 @@ struct Create { mutable_buffer: Option, } -/// Get list of databases, or return configuration of specific database +/// Get list of databases +#[derive(Debug, StructOpt)] +struct List {} + +/// Return configuration of specific database #[derive(Debug, StructOpt)] struct Get { - /// If specified returns configuration of database - name: Option, + /// The name of the database + name: String, } /// Write data into the specified database @@ -103,6 +107,7 @@ struct Query { #[derive(Debug, StructOpt)] enum Command { Create(Create), + List(List), Get(Get), Write(Write), Query(Query), @@ -129,16 +134,16 @@ pub async fn command(url: String, config: Config) -> Result<()> { .await?; println!("Ok"); } + Command::List(_) => { + let mut client = management::Client::new(connection); + let databases = client.list_databases().await?; + println!("{}", databases.join(", ")) + } Command::Get(get) => { let mut client = management::Client::new(connection); - if let Some(name) = get.name { - let database = client.get_database(name).await?; - // TOOD: Do something better than this - println!("{:#?}", database); - } else { - let databases = client.list_databases().await?; - println!("{}", databases.join(", ")) - } + let database = client.get_database(get.name).await?; + // TOOD: Do something better than this + println!("{:#?}", database); } Command::Write(write) => { let mut client = write::Client::new(connection); diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 8ef8688682..2d1ba2597f 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -64,7 +64,7 @@ async fn test_create_database() { Command::cargo_bin("influxdb_iox") .unwrap() .arg("database") - .arg("get") + .arg("list") .arg("--host") .arg(addr) .assert() From 7e25c4e896df07eebde3e308d7dc0bac65bc411b Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Fri, 12 Mar 2021 15:01:27 +0000 Subject: [PATCH 033/104] feat: add fanout task tracking (#956) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- Cargo.lock | 2 + data_types/src/job.rs | 24 + data_types/src/lib.rs | 4 +- generated_types/build.rs | 1 + .../influxdata/iox/management/v1/jobs.proto | 23 + .../iox/management/v1/service.proto | 18 +- server/Cargo.toml | 2 + server/src/buffer.rs | 17 +- server/src/lib.rs | 128 ++-- server/src/tracker.rs | 637 ++++++++++++++---- server/src/tracker/future.rs | 87 +++ server/src/tracker/registry.rs | 126 ++++ src/influxdb_ioxd.rs | 8 +- src/influxdb_ioxd/rpc.rs | 4 +- src/influxdb_ioxd/rpc/management.rs | 10 + src/influxdb_ioxd/rpc/operations.rs | 182 +++++ 16 files changed, 1070 insertions(+), 203 deletions(-) create mode 100644 data_types/src/job.rs create mode 100644 generated_types/protos/influxdata/iox/management/v1/jobs.proto create mode 100644 server/src/tracker/future.rs create mode 100644 server/src/tracker/registry.rs create mode 100644 src/influxdb_ioxd/rpc/operations.rs diff --git a/Cargo.lock b/Cargo.lock index e1101ea91b..323119d758 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3195,6 +3195,7 @@ dependencies = [ "flatbuffers 0.6.1", "futures", "generated_types", + "hashbrown", "influxdb_line_protocol", "mutable_buffer", "object_store", @@ -3208,6 +3209,7 @@ dependencies = [ "snap", "test_helpers", "tokio", + "tokio-util", "tracing", "uuid", ] diff --git a/data_types/src/job.rs b/data_types/src/job.rs new file mode 100644 index 0000000000..738b39a938 --- /dev/null +++ b/data_types/src/job.rs @@ -0,0 +1,24 @@ +use generated_types::influxdata::iox::management::v1 as management; + +/// Metadata associated with a set of background tasks +/// Used in combination with TrackerRegistry +#[derive(Debug, Clone)] +pub enum Job { + PersistSegment { writer_id: u32, segment_id: u64 }, + Dummy { nanos: Vec }, +} + +impl From for management::operation_metadata::Job { + fn from(job: Job) -> Self { + match job { + Job::PersistSegment { + writer_id, + segment_id, + } => Self::PersistSegment(management::PersistSegment { + writer_id, + segment_id, + }), + Job::Dummy { nanos } => Self::Dummy(management::Dummy { nanos }), + } + } +} diff --git a/data_types/src/lib.rs b/data_types/src/lib.rs index c7edc30360..574feb3c27 100644 --- a/data_types/src/lib.rs +++ b/data_types/src/lib.rs @@ -10,6 +10,7 @@ clippy::clone_on_ref_ptr )] +pub use database_name::*; pub use schema::TIME_COLUMN_NAME; /// The name of the column containing table names returned by a call to @@ -25,6 +26,7 @@ pub mod data; pub mod database_rules; pub mod error; pub mod http; +pub mod job; pub mod names; pub mod partition_metadata; pub mod schema; @@ -33,6 +35,4 @@ pub mod timestamp; pub mod wal; mod database_name; -pub use database_name::*; - pub(crate) mod field_validation; diff --git a/generated_types/build.rs b/generated_types/build.rs index 7e66a0f15f..af953aa288 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -41,6 +41,7 @@ fn generate_grpc_types(root: &Path) -> Result<()> { management_path.join("database_rules.proto"), management_path.join("chunk.proto"), management_path.join("service.proto"), + management_path.join("jobs.proto"), write_path.join("service.proto"), root.join("grpc/health/v1/service.proto"), root.join("google/longrunning/operations.proto"), diff --git a/generated_types/protos/influxdata/iox/management/v1/jobs.proto b/generated_types/protos/influxdata/iox/management/v1/jobs.proto new file mode 100644 index 0000000000..01691dbfad --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/jobs.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + +message OperationMetadata { + uint64 cpu_nanos = 1; + uint64 wall_nanos = 2; + uint64 task_count = 3; + uint64 pending_count = 4; + + oneof job { + Dummy dummy = 5; + PersistSegment persist_segment = 6; + } +} + +message PersistSegment { + uint32 writer_id = 1; + uint64 segment_id = 2; +} + +message Dummy { + repeated uint64 nanos = 1; +} diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index e09f9d652f..1a36a8e75a 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package influxdata.iox.management.v1; -import "google/protobuf/empty.proto"; +import "google/longrunning/operations.proto"; import "influxdata/iox/management/v1/database_rules.proto"; import "influxdata/iox/management/v1/chunk.proto"; @@ -27,6 +27,15 @@ service ManagementService { // Delete a reference to remote IOx server. rpc DeleteRemote(DeleteRemoteRequest) returns (DeleteRemoteResponse); + + // Creates a dummy job that for each value of the nanos field + // spawns a task that sleeps for that number of nanoseconds before returning + rpc CreateDummyJob(CreateDummyJobRequest) returns (CreateDummyJobResponse) { + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "OperationMetadata" + }; + } } message GetWriterIdRequest {} @@ -70,6 +79,13 @@ message ListChunksResponse { repeated Chunk chunks = 1; } +message CreateDummyJobRequest { + repeated uint64 nanos = 1; +} + +message CreateDummyJobResponse { + google.longrunning.Operation operation = 1; +} message ListRemotesRequest {} diff --git a/server/Cargo.toml b/server/Cargo.toml index a82d9bd3e6..a68cc4d1fb 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -14,6 +14,7 @@ data_types = { path = "../data_types" } flatbuffers = "0.6" futures = "0.3.7" generated_types = { path = "../generated_types" } +hashbrown = "0.9.1" influxdb_line_protocol = { path = "../influxdb_line_protocol" } mutable_buffer = { path = "../mutable_buffer" } object_store = { path = "../object_store" } @@ -26,6 +27,7 @@ serde_json = "1.0" snafu = "0.6" snap = "1.0.0" tokio = { version = "1.0", features = ["macros", "time"] } +tokio-util = { version = "0.6.3" } tracing = "0.1" uuid = { version = "0.8", features = ["serde", "v4"] } diff --git a/server/src/buffer.rs b/server/src/buffer.rs index 550719b7b4..7046cc1642 100644 --- a/server/src/buffer.rs +++ b/server/src/buffer.rs @@ -15,7 +15,7 @@ use std::{ sync::Arc, }; -use crate::tracker::{TrackedFutureExt, TrackerRegistry}; +use crate::tracker::{TrackedFutureExt, TrackerRegistration}; use bytes::Bytes; use chrono::{DateTime, Utc}; use crc32fast::Hasher; @@ -73,12 +73,6 @@ pub enum Error { InvalidFlatbuffersSegment, } -#[derive(Debug, Clone)] -pub struct SegmentPersistenceTask { - writer_id: u32, - location: object_store::path::Path, -} - pub type Result = std::result::Result; /// An in-memory buffer of a write ahead log. It is split up into segments, @@ -376,7 +370,7 @@ impl Segment { /// the given object store location. pub fn persist_bytes_in_background( &self, - reg: &TrackerRegistry, + tracker: TrackerRegistration, writer_id: u32, db_name: &DatabaseName<'_>, store: Arc, @@ -385,11 +379,6 @@ impl Segment { let location = database_object_store_path(writer_id, db_name, &store); let location = object_store_path_for_segment(&location, self.id)?; - let task_meta = SegmentPersistenceTask { - writer_id, - location: location.clone(), - }; - let len = data.len(); let mut stream_data = std::io::Result::Ok(data.clone()); @@ -414,7 +403,7 @@ impl Segment { // TODO: Mark segment as persisted info!("persisted data to {}", location.display()); } - .track(reg, task_meta), + .track(tracker), ); Ok(()) diff --git a/server/src/lib.rs b/server/src/lib.rs index 495e28ebbf..8b3c7969f0 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -67,43 +67,44 @@ clippy::clone_on_ref_ptr )] -pub mod buffer; -mod config; -pub mod db; -pub mod snapshot; -mod tracker; - -#[cfg(test)] -mod query_tests; - use std::sync::{ atomic::{AtomicU32, Ordering}, Arc, }; -use crate::{ - buffer::SegmentPersistenceTask, - config::{ - object_store_path_for_database_config, Config, GRPCConnectionString, DB_RULES_FILE_NAME, - }, - db::Db, - tracker::TrackerRegistry, -}; +use async_trait::async_trait; +use bytes::Bytes; +use futures::stream::TryStreamExt; +use snafu::{OptionExt, ResultExt, Snafu}; +use tracing::{error, info}; + use data_types::{ data::{lines_to_replicated_write, ReplicatedWrite}, - database_rules::DatabaseRules, + database_rules::{DatabaseRules, WriterId}, + job::Job, {DatabaseName, DatabaseNameError}, }; use influxdb_line_protocol::ParsedLine; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use query::{exec::Executor, DatabaseStore}; -use async_trait::async_trait; -use bytes::Bytes; -use data_types::database_rules::WriterId; -use futures::stream::TryStreamExt; -use snafu::{OptionExt, ResultExt, Snafu}; -use tracing::error; +use crate::tracker::TrackedFutureExt; +use crate::{ + config::{ + object_store_path_for_database_config, Config, GRPCConnectionString, DB_RULES_FILE_NAME, + }, + db::Db, + tracker::{Tracker, TrackerId, TrackerRegistry}, +}; + +pub mod buffer; +mod config; +pub mod db; +pub mod snapshot; +pub mod tracker; + +#[cfg(test)] +mod query_tests; type DatabaseError = Box; @@ -157,7 +158,7 @@ pub struct Server { connection_manager: Arc, pub store: Arc, executor: Arc, - segment_persistence_registry: TrackerRegistry, + jobs: TrackerRegistry, } impl Server { @@ -168,7 +169,7 @@ impl Server { store, connection_manager: Arc::new(connection_manager), executor: Arc::new(Executor::new()), - segment_persistence_registry: TrackerRegistry::new(), + jobs: TrackerRegistry::new(), } } @@ -348,13 +349,14 @@ impl Server { if persist { let writer_id = self.require_id()?; let store = Arc::clone(&self.store); + + let (_, tracker) = self.jobs.register(Job::PersistSegment { + writer_id, + segment_id: segment.id, + }); + segment - .persist_bytes_in_background( - &self.segment_persistence_registry, - writer_id, - db_name, - store, - ) + .persist_bytes_in_background(tracker, writer_id, db_name, store) .context(WalError)?; } } @@ -382,6 +384,50 @@ impl Server { pub fn delete_remote(&self, id: WriterId) -> Option { self.config.delete_remote(id) } + + pub fn spawn_dummy_job(&self, nanos: Vec) -> Tracker { + let (tracker, registration) = self.jobs.register(Job::Dummy { + nanos: nanos.clone(), + }); + + for duration in nanos { + tokio::spawn( + tokio::time::sleep(tokio::time::Duration::from_nanos(duration)) + .track(registration.clone()), + ); + } + + tracker + } + + /// Returns a list of all jobs tracked by this server + pub fn tracked_jobs(&self) -> Vec> { + self.jobs.tracked() + } + + /// Returns a specific job tracked by this server + pub fn get_job(&self, id: TrackerId) -> Option> { + self.jobs.get(id) + } + + /// Background worker function + /// + /// TOOD: Handle termination (#827) + pub async fn background_worker(&self) { + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); + + loop { + // TODO: Retain limited history of past jobs, e.g. enqueue returned data into a + // Dequeue + let reclaimed = self.jobs.reclaim(); + + for job in reclaimed { + info!(?job, "job finished"); + } + + interval.tick().await; + } + } } #[async_trait] @@ -508,20 +554,24 @@ async fn get_store_bytes( #[cfg(test)] mod tests { - use super::*; - use crate::buffer::Segment; - use arrow_deps::{assert_table_eq, datafusion::physical_plan::collect}; + use std::collections::BTreeMap; + use async_trait::async_trait; + use futures::TryStreamExt; + use parking_lot::Mutex; + use snafu::Snafu; + + use arrow_deps::{assert_table_eq, datafusion::physical_plan::collect}; use data_types::database_rules::{ PartitionTemplate, TemplatePart, WalBufferConfig, WalBufferRollover, }; - use futures::TryStreamExt; use influxdb_line_protocol::parse_lines; use object_store::{memory::InMemory, path::ObjectStorePath}; - use parking_lot::Mutex; use query::frontend::sql::SQLQueryPlanner; - use snafu::Snafu; - use std::collections::BTreeMap; + + use crate::buffer::Segment; + + use super::*; type TestError = Box; type Result = std::result::Result; diff --git a/server/src/tracker.rs b/server/src/tracker.rs index 07e0d0d8db..53493a91ba 100644 --- a/server/src/tracker.rs +++ b/server/src/tracker.rs @@ -1,241 +1,592 @@ -use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; +//! This module contains a future tracking system supporting fanout, +//! cancellation and asynchronous signalling of completion +//! +//! A Tracker is created by calling TrackerRegistry::register. TrackedFutures +//! can then be associated with this Tracker and monitored and/or cancelled. +//! +//! This is used within IOx to track futures spawned as multiple tokio tasks. +//! +//! For example, when migrating a chunk from the mutable buffer to the read +//! buffer: +//! +//! - There is a single over-arching Job being performed +//! - A single tracker is allocated from a TrackerRegistry in Server and +//! associated with the Job metadata +//! - This tracker is registered with every future that is spawned as a tokio +//! task +//! +//! This same system may in future form part of a query tracking story +//! +//! # Correctness +//! +//! The key correctness property of the Tracker system is Tracker::is_complete +//! only returns true when all futures associated with the tracker have +//! completed and no more can be spawned. Additionally at such a point +//! all metrics - cpu_nanos, wall_nanos, created_futures should be visible +//! to the calling thread +//! +//! Note: there is no guarantee that pending_registrations or pending_futures +//! ever reaches 0, a program could call mem::forget on a TrackerRegistration, +//! leak the TrackerRegistration, spawn a future that never finishes, etc... +//! Such a program would never consider the Tracker complete and therefore this +//! doesn't violate the correctness property +//! +//! ## Proof +//! +//! 1. pending_registrations starts at 1, and is incremented on +//! TrackerRegistration::clone. A TrackerRegistration cannot be created from an +//! existing TrackerState, only another TrackerRegistration +//! +//! 2. pending_registrations is decremented with release semantics on +//! TrackerRegistration::drop +//! +//! 3. pending_futures is only incremented with a TrackerRegistration in scope +//! +//! 4. 2. + 3. -> A thread that increments pending_futures, decrements +//! pending_registrations with release semantics afterwards. By definition of +//! release semantics these writes to pending_futures cannot be reordered to +//! come after the atomic decrement of pending_registrations +//! +//! 5. 1. + 2. + drop cannot be called multiple times on the same object -> once +//! pending_registrations is decremented to 0 it can never be incremented again +//! +//! 6. 4. + 5. -> the decrement to 0 of pending_registrations must commit after +//! the last increment of pending_futures +//! +//! 7. pending_registrations is loaded with acquire semantics +//! +//! 8. By definition of acquire semantics, any thread that reads +//! pending_registrations is guaranteed to see any increments to pending_futures +//! performed before the most recent decrement of pending_registrations +//! +//! 9. 6. + 8. -> A thread that observes a pending_registrations of 0 cannot +//! subsequently observe pending_futures to increase +//! +//! 10. Tracker::is_complete returns if it observes pending_registrations to be +//! 0 and then pending_futures to be 0 +//! +//! 11. 9 + 10 -> A thread can only observe Tracker::is_complete() == true +//! after all futures have been dropped and no more can be created +//! +//! 12. pending_futures is decremented with Release semantics on +//! TrackedFuture::drop after any associated metrics have been incremented +//! +//! 13. pending_futures is loaded with acquire semantics +//! +//! 14. 12. + 13. -> A thread that observes a pending_futures of 0 is guaranteed +//! to see any metrics from any dropped TrackedFuture +//! +//! Note: this proof ignores the complexity of moving Trackers, TrackedFutures, +//! etc... between threads as any such functionality must perform the necessary +//! synchronisation to be well-formed. + use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::task::{Context, Poll}; +use std::time::Instant; -use futures::prelude::*; -use parking_lot::Mutex; -use pin_project::{pin_project, pinned_drop}; +use tokio_util::sync::CancellationToken; +use tracing::warn; -/// Every future registered with a `TrackerRegistry` is assigned a unique -/// `TrackerId` -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub struct TrackerId(usize); +pub use future::{TrackedFuture, TrackedFutureExt}; +pub use registry::{TrackerId, TrackerRegistry}; +mod future; +mod registry; + +/// The state shared between all sibling tasks #[derive(Debug)] -struct Tracker { - data: T, - abort: future::AbortHandle, +struct TrackerState { + start_instant: Instant, + cancel_token: CancellationToken, + cpu_nanos: AtomicUsize, + wall_nanos: AtomicUsize, + + created_futures: AtomicUsize, + pending_futures: AtomicUsize, + pending_registrations: AtomicUsize, + + watch: tokio::sync::watch::Receiver, } +/// A Tracker can be used to monitor/cancel/wait for a set of associated futures #[derive(Debug)] -struct TrackerContextInner { - id: AtomicUsize, - trackers: Mutex>>, +pub struct Tracker { + id: TrackerId, + state: Arc, + metadata: Arc, } -/// Allows tracking the lifecycle of futures registered by -/// `TrackedFutureExt::track` with an accompanying metadata payload of type T -/// -/// Additionally can trigger graceful termination of registered futures -#[derive(Debug)] -pub struct TrackerRegistry { - inner: Arc>, -} - -// Manual Clone to workaround https://github.com/rust-lang/rust/issues/26925 -impl Clone for TrackerRegistry { +impl Clone for Tracker { fn clone(&self) -> Self { Self { - inner: Arc::clone(&self.inner), + id: self.id, + state: Arc::clone(&self.state), + metadata: Arc::clone(&self.metadata), } } } -impl Default for TrackerRegistry { - fn default() -> Self { - Self { - inner: Arc::new(TrackerContextInner { - id: AtomicUsize::new(0), - trackers: Mutex::new(Default::default()), - }), - } - } -} - -impl TrackerRegistry { - pub fn new() -> Self { - Default::default() +impl Tracker { + /// Returns the ID of the Tracker - these are unique per TrackerRegistry + pub fn id(&self) -> TrackerId { + self.id } - /// Trigger graceful termination of a registered future - /// - /// Returns false if no future found with the provided ID + /// Returns a reference to the metadata stored within this Tracker + pub fn metadata(&self) -> &T { + &self.metadata + } + + /// Trigger graceful termination of any futures tracked by + /// this tracker /// /// Note: If the future is currently executing, termination /// will only occur when the future yields (returns from poll) - #[allow(dead_code)] - pub fn terminate(&self, id: TrackerId) -> bool { - if let Some(meta) = self.inner.trackers.lock().get_mut(&id) { - meta.abort.abort(); - true - } else { - false + /// and is then scheduled to run again + pub fn cancel(&self) { + self.state.cancel_token.cancel(); + } + + /// Returns the number of outstanding futures + pub fn pending_futures(&self) -> usize { + self.state.pending_futures.load(Ordering::Relaxed) + } + + /// Returns the number of TrackedFutures created with this Tracker + pub fn created_futures(&self) -> usize { + self.state.created_futures.load(Ordering::Relaxed) + } + + /// Returns the number of nanoseconds futures tracked by this + /// tracker have spent executing + pub fn cpu_nanos(&self) -> usize { + self.state.cpu_nanos.load(Ordering::Relaxed) + } + + /// Returns the number of nanoseconds since the Tracker was registered + /// to the time the last TrackedFuture was dropped + /// + /// Returns 0 if there are still pending tasks + pub fn wall_nanos(&self) -> usize { + if !self.is_complete() { + return 0; } + self.state.wall_nanos.load(Ordering::Relaxed) } - fn untrack(&self, id: &TrackerId) { - self.inner.trackers.lock().remove(id); + /// Returns true if all futures associated with this tracker have + /// been dropped and no more can be created + pub fn is_complete(&self) -> bool { + // The atomic decrement in TrackerRegistration::drop has release semantics + // acquire here ensures that if a thread observes the tracker to have + // no pending_registrations it cannot subsequently observe pending_futures + // to increase. If it could, observing pending_futures==0 would be insufficient + // to conclude there are no outstanding futures + let pending_registrations = self.state.pending_registrations.load(Ordering::Acquire); + + // The atomic decrement in TrackedFuture::drop has release semantics + // acquire therefore ensures that if a thread observes the completion of + // a TrackedFuture, it is guaranteed to see its updates (e.g. wall_nanos) + let pending_futures = self.state.pending_futures.load(Ordering::Acquire); + + pending_registrations == 0 && pending_futures == 0 } - fn track(&self, metadata: T) -> (TrackerId, future::AbortRegistration) { - let id = TrackerId(self.inner.id.fetch_add(1, Ordering::Relaxed)); - let (abort_handle, abort_registration) = future::AbortHandle::new_pair(); - - self.inner.trackers.lock().insert( - id, - Tracker { - abort: abort_handle, - data: metadata, - }, - ); - - (id, abort_registration) + /// Returns if this tracker has been cancelled + pub fn is_cancelled(&self) -> bool { + self.state.cancel_token.is_cancelled() } -} -impl TrackerRegistry { - /// Returns a list of tracked futures, with their accompanying IDs and - /// metadata - #[allow(dead_code)] - pub fn tracked(&self) -> Vec<(TrackerId, T)> { - // TODO: Improve this - (#711) - self.inner - .trackers - .lock() - .iter() - .map(|(id, value)| (*id, value.data.clone())) - .collect() - } -} + /// Blocks until all futures associated with the tracker have been + /// dropped and no more can be created + pub async fn join(&self) { + let mut watch = self.state.watch.clone(); -/// An extension trait that provides `self.track(reg, {})` allowing -/// registering this future with a `TrackerRegistry` -pub trait TrackedFutureExt: Future { - fn track(self, reg: &TrackerRegistry, metadata: T) -> TrackedFuture - where - Self: Sized, - { - let (id, registration) = reg.track(metadata); - - TrackedFuture { - inner: future::Abortable::new(self, registration), - reg: reg.clone(), - id, + // Wait until watch is set to true or the tx side is dropped + while !*watch.borrow() { + if watch.changed().await.is_err() { + // tx side has been dropped + warn!("tracker watch dropped without signalling"); + break; + } } } } -impl TrackedFutureExt for T where T: Future {} - -/// The `Future` returned by `TrackedFutureExt::track()` -/// Unregisters the future from the registered `TrackerRegistry` on drop -/// and provides the early termination functionality used by -/// `TrackerRegistry::terminate` -#[pin_project(PinnedDrop)] -pub struct TrackedFuture { - #[pin] - inner: future::Abortable, - - reg: TrackerRegistry, - id: TrackerId, +/// A TrackerRegistration is returned by TrackerRegistry::register and can be +/// used to register new TrackedFutures +/// +/// A tracker will not be considered completed until all TrackerRegistrations +/// referencing it have been dropped. This is to prevent a race where further +/// TrackedFutures are registered with a Tracker that has already signalled +/// completion +#[derive(Debug)] +pub struct TrackerRegistration { + state: Arc, } -impl Future for TrackedFuture { - type Output = Result; +impl Clone for TrackerRegistration { + fn clone(&self) -> Self { + self.state + .pending_registrations + .fetch_add(1, Ordering::Relaxed); - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.project().inner.poll(cx) + Self { + state: Arc::clone(&self.state), + } } } -#[pinned_drop] -impl PinnedDrop for TrackedFuture { - fn drop(self: Pin<&mut Self>) { - // Note: This could cause a double-panic in an extreme situation where - // the internal `TrackerRegistry` lock is poisoned and drop was - // called as part of unwinding the stack to handle another panic - let this = self.project(); - this.reg.untrack(this.id) +impl TrackerRegistration { + fn new(watch: tokio::sync::watch::Receiver) -> Self { + let state = Arc::new(TrackerState { + start_instant: Instant::now(), + cpu_nanos: AtomicUsize::new(0), + wall_nanos: AtomicUsize::new(0), + cancel_token: CancellationToken::new(), + created_futures: AtomicUsize::new(0), + pending_futures: AtomicUsize::new(0), + pending_registrations: AtomicUsize::new(1), + watch, + }); + + Self { state } + } +} + +impl Drop for TrackerRegistration { + fn drop(&mut self) { + let previous = self + .state + .pending_registrations + .fetch_sub(1, Ordering::Release); + assert_ne!(previous, 0); } } #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; use tokio::sync::oneshot; #[tokio::test] async fn test_lifecycle() { let (sender, receive) = oneshot::channel(); - let reg = TrackerRegistry::new(); + let registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); - let task = tokio::spawn(receive.track(®, ())); + let task = tokio::spawn(receive.track(registration)); - assert_eq!(reg.tracked().len(), 1); + assert_eq!(registry.running().len(), 1); sender.send(()).unwrap(); task.await.unwrap().unwrap().unwrap(); - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); } #[tokio::test] async fn test_interleaved() { let (sender1, receive1) = oneshot::channel(); let (sender2, receive2) = oneshot::channel(); - let reg = TrackerRegistry::new(); + let registry = TrackerRegistry::new(); + let (_, registration1) = registry.register(1); + let (_, registration2) = registry.register(2); - let task1 = tokio::spawn(receive1.track(®, 1)); - let task2 = tokio::spawn(receive2.track(®, 2)); + let task1 = tokio::spawn(receive1.track(registration1)); + let task2 = tokio::spawn(receive2.track(registration2)); - let mut tracked: Vec<_> = reg.tracked().iter().map(|x| x.1).collect(); - tracked.sort_unstable(); - assert_eq!(tracked, vec![1, 2]); + let tracked = sorted(registry.running()); + assert_eq!(get_metadata(&tracked), vec![1, 2]); sender2.send(()).unwrap(); task2.await.unwrap().unwrap().unwrap(); - let tracked: Vec<_> = reg.tracked().iter().map(|x| x.1).collect(); - assert_eq!(tracked, vec![1]); + let tracked: Vec<_> = sorted(registry.running()); + assert_eq!(get_metadata(&tracked), vec![1]); sender1.send(42).unwrap(); let ret = task1.await.unwrap().unwrap().unwrap(); assert_eq!(ret, 42); - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); } #[tokio::test] async fn test_drop() { - let reg = TrackerRegistry::new(); + let registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); { - let f = futures::future::pending::<()>().track(®, ()); + let f = futures::future::pending::<()>().track(registration); - assert_eq!(reg.tracked().len(), 1); + assert_eq!(registry.running().len(), 1); std::mem::drop(f); } - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_drop_multiple() { + let registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); + + { + let f = futures::future::pending::<()>().track(registration.clone()); + { + let f = futures::future::pending::<()>().track(registration); + assert_eq!(registry.running().len(), 1); + std::mem::drop(f); + } + assert_eq!(registry.running().len(), 1); + std::mem::drop(f); + } + + assert_eq!(registry.running().len(), 0); } #[tokio::test] async fn test_terminate() { - let reg = TrackerRegistry::new(); + let registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); - let task = tokio::spawn(futures::future::pending::<()>().track(®, ())); + let task = tokio::spawn(futures::future::pending::<()>().track(registration)); - let tracked = reg.tracked(); + let tracked = registry.running(); assert_eq!(tracked.len(), 1); - reg.terminate(tracked[0].0); + tracked[0].cancel(); let result = task.await.unwrap(); assert!(result.is_err()); - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_terminate_early() { + let registry = TrackerRegistry::new(); + let (tracker, registration) = registry.register(()); + tracker.cancel(); + + let task1 = tokio::spawn(futures::future::pending::<()>().track(registration)); + let result1 = task1.await.unwrap(); + + assert!(result1.is_err()); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_terminate_multiple() { + let registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); + + let task1 = tokio::spawn(futures::future::pending::<()>().track(registration.clone())); + let task2 = tokio::spawn(futures::future::pending::<()>().track(registration)); + + let tracked = registry.running(); + assert_eq!(tracked.len(), 1); + + tracked[0].cancel(); + + let result1 = task1.await.unwrap(); + let result2 = task2.await.unwrap(); + + assert!(result1.is_err()); + assert!(result2.is_err()); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_reclaim() { + let registry = TrackerRegistry::new(); + + let (_, registration1) = registry.register(1); + let (_, registration2) = registry.register(2); + let (_, registration3) = registry.register(3); + + let task1 = tokio::spawn(futures::future::pending::<()>().track(registration1.clone())); + let task2 = tokio::spawn(futures::future::pending::<()>().track(registration1)); + let task3 = tokio::spawn(futures::future::ready(()).track(registration2.clone())); + let task4 = tokio::spawn(futures::future::pending::<()>().track(registration2)); + let task5 = tokio::spawn(futures::future::pending::<()>().track(registration3)); + + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 3); + assert_eq!(get_metadata(&running), vec![1, 2, 3]); + assert_eq!(tracked.len(), 3); + assert_eq!(get_metadata(&tracked), vec![1, 2, 3]); + + // Trigger termination of task1 and task2 + running[0].cancel(); + + let result1 = task1.await.unwrap(); + let result2 = task2.await.unwrap(); + + assert!(result1.is_err()); + assert!(result2.is_err()); + + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 2); + assert_eq!(get_metadata(&running), vec![2, 3]); + assert_eq!(tracked.len(), 3); + assert_eq!(get_metadata(&tracked), vec![1, 2, 3]); + + // Expect reclaim to find now finished registration1 + let reclaimed = sorted(registry.reclaim()); + assert_eq!(reclaimed.len(), 1); + assert_eq!(get_metadata(&reclaimed), vec![1]); + + // Now expect tracked to match running + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 2); + assert_eq!(get_metadata(&running), vec![2, 3]); + assert_eq!(tracked.len(), 2); + assert_eq!(get_metadata(&tracked), vec![2, 3]); + + // Wait for task3 to finish + let result3 = task3.await.unwrap(); + assert!(result3.is_ok()); + + assert_eq!(tracked[0].pending_futures(), 1); + assert_eq!(tracked[0].created_futures(), 2); + assert!(!tracked[0].is_complete()); + + // Trigger termination of task5 + running[1].cancel(); + + let result5 = task5.await.unwrap(); + assert!(result5.is_err()); + + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 1); + assert_eq!(get_metadata(&running), vec![2]); + assert_eq!(tracked.len(), 2); + assert_eq!(get_metadata(&tracked), vec![2, 3]); + + // Trigger termination of task4 + running[0].cancel(); + + let result4 = task4.await.unwrap(); + assert!(result4.is_err()); + + assert_eq!(running[0].pending_futures(), 0); + assert_eq!(running[0].created_futures(), 2); + assert!(running[0].is_complete()); + + let reclaimed = sorted(registry.reclaim()); + + assert_eq!(reclaimed.len(), 2); + assert_eq!(get_metadata(&reclaimed), vec![2, 3]); + assert_eq!(registry.tracked().len(), 0); + } + + // Use n+1 threads where n is the number of "blocking" tasks + // to prevent stalling the tokio executor + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_timing() { + let registry = TrackerRegistry::new(); + let (tracker1, registration1) = registry.register(1); + let (tracker2, registration2) = registry.register(2); + let (tracker3, registration3) = registry.register(3); + + let task1 = + tokio::spawn(tokio::time::sleep(Duration::from_millis(100)).track(registration1)); + let task2 = tokio::spawn( + async move { std::thread::sleep(Duration::from_millis(100)) }.track(registration2), + ); + + let task3 = tokio::spawn( + async move { std::thread::sleep(Duration::from_millis(100)) } + .track(registration3.clone()), + ); + + let task4 = tokio::spawn( + async move { std::thread::sleep(Duration::from_millis(100)) }.track(registration3), + ); + + task1.await.unwrap().unwrap(); + task2.await.unwrap().unwrap(); + task3.await.unwrap().unwrap(); + task4.await.unwrap().unwrap(); + + assert_eq!(tracker1.pending_futures(), 0); + assert_eq!(tracker2.pending_futures(), 0); + assert_eq!(tracker3.pending_futures(), 0); + + assert!(tracker1.is_complete()); + assert!(tracker2.is_complete()); + assert!(tracker3.is_complete()); + + assert_eq!(tracker2.created_futures(), 1); + assert_eq!(tracker2.created_futures(), 1); + assert_eq!(tracker3.created_futures(), 2); + + let assert_fuzzy = |actual: usize, expected: std::time::Duration| { + // Number of milliseconds of toleration + let epsilon = Duration::from_millis(10).as_nanos() as usize; + let expected = expected.as_nanos() as usize; + + assert!( + actual > expected.saturating_sub(epsilon), + "Expected {} got {}", + expected, + actual + ); + assert!( + actual < expected.saturating_add(epsilon), + "Expected {} got {}", + expected, + actual + ); + }; + + assert_fuzzy(tracker1.cpu_nanos(), Duration::from_millis(0)); + assert_fuzzy(tracker1.wall_nanos(), Duration::from_millis(100)); + assert_fuzzy(tracker2.cpu_nanos(), Duration::from_millis(100)); + assert_fuzzy(tracker2.wall_nanos(), Duration::from_millis(100)); + assert_fuzzy(tracker3.cpu_nanos(), Duration::from_millis(200)); + assert_fuzzy(tracker3.wall_nanos(), Duration::from_millis(100)); + } + + #[tokio::test] + async fn test_register_race() { + let registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); + + let task1 = tokio::spawn(futures::future::ready(()).track(registration.clone())); + task1.await.unwrap().unwrap(); + + // Should only consider tasks complete once cannot register more Futures + let reclaimed = registry.reclaim(); + assert_eq!(reclaimed.len(), 0); + + let task2 = tokio::spawn(futures::future::ready(()).track(registration)); + task2.await.unwrap().unwrap(); + + let reclaimed = registry.reclaim(); + assert_eq!(reclaimed.len(), 1); + } + + fn sorted(mut input: Vec>) -> Vec> { + input.sort_unstable_by_key(|x| *x.metadata()); + input + } + + fn get_metadata(input: &[Tracker]) -> Vec { + let mut ret: Vec<_> = input.iter().map(|x| *x.metadata()).collect(); + ret.sort_unstable(); + ret } } diff --git a/server/src/tracker/future.rs b/server/src/tracker/future.rs new file mode 100644 index 0000000000..342a89adb2 --- /dev/null +++ b/server/src/tracker/future.rs @@ -0,0 +1,87 @@ +use std::pin::Pin; +use std::sync::atomic::Ordering; +use std::task::{Context, Poll}; +use std::time::Instant; + +use futures::{future::BoxFuture, prelude::*}; +use pin_project::{pin_project, pinned_drop}; + +use super::{TrackerRegistration, TrackerState}; +use std::sync::Arc; + +/// An extension trait that provides `self.track(registration)` allowing +/// associating this future with a `TrackerRegistration` +pub trait TrackedFutureExt: Future { + fn track(self, registration: TrackerRegistration) -> TrackedFuture + where + Self: Sized, + { + let tracker = Arc::clone(®istration.state); + let token = tracker.cancel_token.clone(); + + tracker.created_futures.fetch_add(1, Ordering::Relaxed); + tracker.pending_futures.fetch_add(1, Ordering::Relaxed); + + // This must occur after the increment of pending_futures + std::mem::drop(registration); + + // The future returned by CancellationToken::cancelled borrows the token + // In order to ensure we get a future with a static lifetime + // we box them up together and let async work its magic + let abort = Box::pin(async move { token.cancelled().await }); + + TrackedFuture { + inner: self, + abort, + tracker, + } + } +} + +impl TrackedFutureExt for T where T: Future {} + +/// The `Future` returned by `TrackedFutureExt::track()` +/// Unregisters the future from the registered `TrackerRegistry` on drop +/// and provides the early termination functionality used by +/// `TrackerRegistry::terminate` +#[pin_project(PinnedDrop)] +#[allow(missing_debug_implementations)] +pub struct TrackedFuture { + #[pin] + inner: F, + #[pin] + abort: BoxFuture<'static, ()>, + tracker: Arc, +} + +impl Future for TrackedFuture { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.as_mut().project().abort.poll(cx).is_ready() { + return Poll::Ready(Err(future::Aborted {})); + } + + let start = Instant::now(); + let poll = self.as_mut().project().inner.poll(cx); + let delta = start.elapsed().as_nanos() as usize; + + self.tracker.cpu_nanos.fetch_add(delta, Ordering::Relaxed); + + poll.map(Ok) + } +} + +#[pinned_drop] +impl PinnedDrop for TrackedFuture { + fn drop(self: Pin<&mut Self>) { + let state = &self.project().tracker; + + let wall_nanos = state.start_instant.elapsed().as_nanos() as usize; + + state.wall_nanos.fetch_max(wall_nanos, Ordering::Relaxed); + + let previous = state.pending_futures.fetch_sub(1, Ordering::Release); + assert_ne!(previous, 0); + } +} diff --git a/server/src/tracker/registry.rs b/server/src/tracker/registry.rs new file mode 100644 index 0000000000..88028269c6 --- /dev/null +++ b/server/src/tracker/registry.rs @@ -0,0 +1,126 @@ +use super::{Tracker, TrackerRegistration}; +use hashbrown::HashMap; +use parking_lot::Mutex; +use std::str::FromStr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use tracing::debug; + +/// Every future registered with a `TrackerRegistry` is assigned a unique +/// `TrackerId` +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct TrackerId(usize); + +impl FromStr for TrackerId { + type Err = std::num::ParseIntError; + + fn from_str(s: &str) -> Result { + Ok(Self(FromStr::from_str(s)?)) + } +} + +impl ToString for TrackerId { + fn to_string(&self) -> String { + self.0.to_string() + } +} + +/// Internal data stored by TrackerRegistry +#[derive(Debug)] +struct TrackerSlot { + tracker: Tracker, + watch: tokio::sync::watch::Sender, +} + +/// Allows tracking the lifecycle of futures registered by +/// `TrackedFutureExt::track` with an accompanying metadata payload of type T +/// +/// Additionally can trigger graceful cancellation of registered futures +#[derive(Debug)] +pub struct TrackerRegistry { + next_id: AtomicUsize, + trackers: Mutex>>, +} + +impl Default for TrackerRegistry { + fn default() -> Self { + Self { + next_id: AtomicUsize::new(0), + trackers: Default::default(), + } + } +} + +impl TrackerRegistry { + pub fn new() -> Self { + Default::default() + } + + /// Register a new tracker in the registry + pub fn register(&self, metadata: T) -> (Tracker, TrackerRegistration) { + let id = TrackerId(self.next_id.fetch_add(1, Ordering::Relaxed)); + let (sender, receiver) = tokio::sync::watch::channel(false); + let registration = TrackerRegistration::new(receiver); + + let tracker = Tracker { + id, + metadata: Arc::new(metadata), + state: Arc::clone(®istration.state), + }; + + self.trackers.lock().insert( + id, + TrackerSlot { + tracker: tracker.clone(), + watch: sender, + }, + ); + + (tracker, registration) + } + + /// Removes completed tasks from the registry and returns a list of those + /// removed + pub fn reclaim(&self) -> Vec> { + self.trackers + .lock() + .drain_filter(|_, v| v.tracker.is_complete()) + .map(|(_, v)| { + if let Err(error) = v.watch.send(true) { + // As we hold a reference to the Tracker here, this should be impossible + debug!(?error, "failed to publish tracker completion") + } + v.tracker + }) + .collect() + } +} + +impl TrackerRegistry { + pub fn get(&self, id: TrackerId) -> Option> { + self.trackers.lock().get(&id).map(|x| x.tracker.clone()) + } + + /// Returns a list of trackers, including those that are no longer running + pub fn tracked(&self) -> Vec> { + self.trackers + .lock() + .iter() + .map(|(_, v)| v.tracker.clone()) + .collect() + } + + /// Returns a list of active trackers + pub fn running(&self) -> Vec> { + self.trackers + .lock() + .iter() + .filter_map(|(_, v)| { + if !v.tracker.is_complete() { + return Some(v.tracker.clone()); + } + None + }) + .collect() + } +} diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index 45b0c57bb9..f8f41b6a60 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -130,7 +130,6 @@ pub async fn main(logging_level: LoggingLevel, config: RunConfig) -> Result<()> info!(bind_address=?grpc_bind_addr, "gRPC server listening"); // Construct and start up HTTP server - let router_service = http::router_service(Arc::clone(&app_server)); let bind_addr = config.http_bind_address; @@ -142,8 +141,11 @@ pub async fn main(logging_level: LoggingLevel, config: RunConfig) -> Result<()> let git_hash = option_env!("GIT_HASH").unwrap_or("UNKNOWN"); info!(git_hash, "InfluxDB IOx server ready"); - // Wait for both the servers to complete - let (grpc_server, server) = futures::future::join(grpc_server, http_server).await; + // Get IOx background worker task + let app = app_server.background_worker(); + + // TODO: Fix shutdown handling (#827) + let (grpc_server, server, _) = futures::future::join3(grpc_server, http_server, app).await; grpc_server.context(ServingRPC)?; server.context(ServingHttp)?; diff --git a/src/influxdb_ioxd/rpc.rs b/src/influxdb_ioxd/rpc.rs index 7998674235..385071ffb9 100644 --- a/src/influxdb_ioxd/rpc.rs +++ b/src/influxdb_ioxd/rpc.rs @@ -11,6 +11,7 @@ use server::{ConnectionManager, Server}; pub mod error; mod flight; mod management; +mod operations; mod storage; mod testing; mod write; @@ -53,7 +54,8 @@ where .add_service(storage::make_server(Arc::clone(&server))) .add_service(flight::make_server(Arc::clone(&server))) .add_service(write::make_server(Arc::clone(&server))) - .add_service(management::make_server(server)) + .add_service(management::make_server(Arc::clone(&server))) + .add_service(operations::make_server(server)) .serve_with_incoming(stream) .await .context(ServerError {}) diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index 570d57875a..fef16299d6 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -127,6 +127,16 @@ where Ok(Response::new(ListChunksResponse { chunks })) } + async fn create_dummy_job( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let slot = self.server.spawn_dummy_job(request.nanos); + let operation = Some(super::operations::encode_tracker(slot)?); + Ok(Response::new(CreateDummyJobResponse { operation })) + } + async fn list_remotes( &self, _: Request, diff --git a/src/influxdb_ioxd/rpc/operations.rs b/src/influxdb_ioxd/rpc/operations.rs new file mode 100644 index 0000000000..9bacc595ae --- /dev/null +++ b/src/influxdb_ioxd/rpc/operations.rs @@ -0,0 +1,182 @@ +use std::fmt::Debug; +use std::sync::Arc; + +use bytes::BytesMut; +use prost::Message; +use tonic::Response; +use tracing::debug; + +use data_types::job::Job; +use generated_types::google::FieldViolationExt; +use generated_types::{ + google::{ + longrunning::*, + protobuf::{Any, Empty}, + rpc::Status, + FieldViolation, InternalError, NotFound, + }, + influxdata::iox::management::v1 as management, +}; +use server::{ + tracker::{Tracker, TrackerId}, + ConnectionManager, Server, +}; +use std::convert::TryInto; + +/// Implementation of the write service +struct OperationsService { + server: Arc>, +} + +pub fn encode_tracker(tracker: Tracker) -> Result { + let id = tracker.id(); + let is_complete = tracker.is_complete(); + let is_cancelled = tracker.is_cancelled(); + + let mut buffer = BytesMut::new(); + management::OperationMetadata { + cpu_nanos: tracker.cpu_nanos() as _, + wall_nanos: tracker.wall_nanos() as _, + task_count: tracker.created_futures() as _, + pending_count: tracker.pending_futures() as _, + job: Some(tracker.metadata().clone().into()), + } + .encode(&mut buffer) + .map_err(|error| { + debug!(?error, "Unexpected error"); + InternalError {} + })?; + + let metadata = Any { + type_url: "type.googleapis.com/influxdata.iox.management.v1.OperationMetadata".to_string(), + value: buffer.freeze(), + }; + + let result = match (is_complete, is_cancelled) { + (true, true) => Some(operation::Result::Error(Status { + code: tonic::Code::Cancelled as _, + message: "Job cancelled".to_string(), + details: vec![], + })), + + (true, false) => Some(operation::Result::Response(Any { + type_url: "type.googleapis.com/google.protobuf.Empty".to_string(), + value: Default::default(), // TODO: Verify this is correct + })), + + _ => None, + }; + + Ok(Operation { + name: id.to_string(), + metadata: Some(metadata), + done: is_complete, + result, + }) +} + +fn get_tracker(server: &Server, tracker: String) -> Result, tonic::Status> +where + M: ConnectionManager, +{ + let tracker_id = tracker.parse::().map_err(|e| FieldViolation { + field: "name".to_string(), + description: e.to_string(), + })?; + + let tracker = server.get_job(tracker_id).ok_or(NotFound { + resource_type: "job".to_string(), + resource_name: tracker, + ..Default::default() + })?; + + Ok(tracker) +} + +#[tonic::async_trait] +impl operations_server::Operations for OperationsService +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + async fn list_operations( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + // TODO: Support pagination + let operations: Result, _> = self + .server + .tracked_jobs() + .into_iter() + .map(encode_tracker) + .collect(); + + Ok(Response::new(ListOperationsResponse { + operations: operations?, + next_page_token: Default::default(), + })) + } + + async fn get_operation( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); + let tracker = get_tracker(self.server.as_ref(), request.name)?; + + Ok(Response::new(encode_tracker(tracker)?)) + } + + async fn delete_operation( + &self, + _: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented( + "IOx does not support operation deletion", + )) + } + + async fn cancel_operation( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); + + let tracker = get_tracker(self.server.as_ref(), request.name)?; + tracker.cancel(); + + Ok(Response::new(Empty {})) + } + + async fn wait_operation( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + // This should take into account the context deadline timeout + // Unfortunately these are currently stripped by tonic + // - https://github.com/hyperium/tonic/issues/75 + + let request = request.into_inner(); + + let tracker = get_tracker(self.server.as_ref(), request.name)?; + if let Some(timeout) = request.timeout { + let timeout = timeout.try_into().field("timeout")?; + + // Timeout is not an error so suppress it + let _ = tokio::time::timeout(timeout, tracker.join()).await; + } else { + tracker.join().await; + } + + Ok(Response::new(encode_tracker(tracker)?)) + } +} + +/// Instantiate the write service +pub fn make_server( + server: Arc>, +) -> operations_server::OperationsServer +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + operations_server::OperationsServer::new(OperationsService { server }) +} From 334fb149b1ba9351799b61e7d78de7817700cba3 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Fri, 12 Mar 2021 18:30:49 +0100 Subject: [PATCH 034/104] feat: Rename server run command to just run Closes #976 --- Dockerfile | 2 +- README.md | 4 +- docker/Dockerfile.iox | 2 +- docs/env.example | 2 +- docs/testing.md | 4 +- src/commands/logging.rs | 4 +- src/commands/run.rs | 341 +++++++++++++++++++++++++++++++++ src/commands/server.rs | 332 +------------------------------- src/influxdb_ioxd.rs | 28 +-- src/main.rs | 21 +- tests/common/server_fixture.rs | 2 - 11 files changed, 381 insertions(+), 361 deletions(-) create mode 100644 src/commands/run.rs diff --git a/Dockerfile b/Dockerfile index 84535ce2d8..46b9d2a192 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,4 +37,4 @@ EXPOSE 8080 8082 ENTRYPOINT ["/usr/bin/influxdb_iox"] -CMD ["server", "run"] +CMD ["run"] diff --git a/README.md b/README.md index 6decf9931a..1468a42a4c 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ takes its configuration as environment variables. You can see a list of the current configuration values by running `influxdb_iox --help`, as well as the specific subcommand config options such as `influxdb_iox -server --help`. +run --help`. Should you desire specifying config via a file, you can do so using a `.env` formatted file in the working directory. You can use the @@ -175,7 +175,7 @@ cargo build --release which will create the corresponding binary in `target/release`: ```shell -./target/release/influxdb_iox server +./target/release/influxdb_iox run ``` Similarly, you can do this in one step with: diff --git a/docker/Dockerfile.iox b/docker/Dockerfile.iox index f75d76ec19..ffa68a7092 100644 --- a/docker/Dockerfile.iox +++ b/docker/Dockerfile.iox @@ -21,4 +21,4 @@ EXPOSE 8080 8082 ENTRYPOINT ["influxdb_iox"] -CMD ["server", "run"] +CMD ["run"] diff --git a/docs/env.example b/docs/env.example index ed4f28a5c6..dc250a3530 100644 --- a/docs/env.example +++ b/docs/env.example @@ -7,7 +7,7 @@ # The full list of available configuration values can be found by in # the command line help (e.g. `env: INFLUXDB_IOX_DB_DIR=`): # -# ./influxdb_iox server --help +# ./influxdb_iox run --help # # # The identifier for the server. Used for writing to object storage and as diff --git a/docs/testing.md b/docs/testing.md index 30c936dfd9..9e424127ec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,14 +21,14 @@ of the object stores, the relevant tests will run. ### Configuration differences when running the tests -When running `influxdb_iox server`, you can pick one object store to use. When running the tests, +When running `influxdb_iox run`, you can pick one object store to use. When running the tests, you can run them against all the possible object stores. There's still only one `INFLUXDB_IOX_BUCKET` variable, though, so that will set the bucket name for all configured object stores. Use the same bucket name when setting up the different services. Other than possibly configuring multiple object stores, configuring the tests to use the object store services is the same as configuring the server to use an object store service. See the output -of `influxdb_iox server --help` for instructions. +of `influxdb_iox run --help` for instructions. ## InfluxDB IOx Client diff --git a/src/commands/logging.rs b/src/commands/logging.rs index bc0386431f..0cd4ae0de3 100644 --- a/src/commands/logging.rs +++ b/src/commands/logging.rs @@ -2,7 +2,7 @@ use tracing_subscriber::{prelude::*, EnvFilter}; -use super::server::{LogFormat, RunConfig}; +use super::run::{Config, LogFormat}; /// Handles setting up logging levels #[derive(Debug)] @@ -81,7 +81,7 @@ impl LoggingLevel { /// Configures logging and tracing, based on the configuration /// values, for the IOx server (the whole enchalada) - pub fn setup_logging(&self, config: &RunConfig) -> Option { + pub fn setup_logging(&self, config: &Config) -> Option { // Copy anything from the config to the rust log environment self.set_rust_log_if_needed(config.rust_log.clone()); diff --git a/src/commands/run.rs b/src/commands/run.rs new file mode 100644 index 0000000000..ac33042f14 --- /dev/null +++ b/src/commands/run.rs @@ -0,0 +1,341 @@ +//! Implementation of command line option for running server + +use crate::commands::logging::LoggingLevel; +use crate::influxdb_ioxd; +use clap::arg_enum; +use std::{net::SocketAddr, net::ToSocketAddrs, path::PathBuf}; +use structopt::StructOpt; +use thiserror::Error; + +/// The default bind address for the HTTP API. +pub const DEFAULT_API_BIND_ADDR: &str = "127.0.0.1:8080"; + +/// The default bind address for the gRPC. +pub const DEFAULT_GRPC_BIND_ADDR: &str = "127.0.0.1:8082"; + +/// The AWS region to use for Amazon S3 based object storage if none is +/// specified. +pub const FALLBACK_AWS_REGION: &str = "us-east-1"; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Run: {0}")] + ServerError(#[from] influxdb_ioxd::Error), +} + +pub type Result = std::result::Result; + +#[derive(Debug, StructOpt)] +#[structopt( + name = "run", + about = "Runs in server mode", + long_about = "Run the IOx server.\n\nThe configuration options below can be \ + set either with the command line flags or with the specified environment \ + variable. If there is a file named '.env' in the current working directory, \ + it is sourced before loading the configuration. + +Configuration is loaded from the following sources (highest precedence first): + - command line arguments + - user set environment variables + - .env file contents + - pre-configured default values" +)] +pub struct Config { + /// This controls the IOx server logging level, as described in + /// https://crates.io/crates/env_logger. + /// + /// Levels for different modules can be specified as well. For example + /// `debug,hyper::proto::h1=info` specifies debug logging for all modules + /// except for the `hyper::proto::h1' module which will only display info + /// level logging. + #[structopt(long = "--log", env = "RUST_LOG")] + pub rust_log: Option, + + /// Log message format. Can be one of: + /// + /// "rust" (default) + /// "logfmt" (logfmt/Heroku style - https://brandur.org/logfmt) + #[structopt(long = "--log_format", env = "INFLUXDB_IOX_LOG_FORMAT")] + pub log_format: Option, + + /// This sets logging up with a pre-configured set of convenient log levels. + /// + /// -v means 'info' log levels + /// -vv means 'verbose' log level (with the exception of some particularly + /// low level libraries) + /// + /// This option is ignored if --log / RUST_LOG are set + #[structopt( + short = "-v", + long = "--verbose", + multiple = true, + takes_value = false, + parse(from_occurrences) + )] + pub verbose_count: u64, + + /// The identifier for the server. + /// + /// Used for writing to object storage and as an identifier that is added to + /// replicated writes, WAL segments and Chunks. Must be unique in a group of + /// connected or semi-connected IOx servers. Must be a number that can be + /// represented by a 32-bit unsigned integer. + #[structopt(long = "--writer-id", env = "INFLUXDB_IOX_ID")] + pub writer_id: Option, + + /// The address on which IOx will serve HTTP API requests. + #[structopt( + long = "--api-bind", + env = "INFLUXDB_IOX_BIND_ADDR", + default_value = DEFAULT_API_BIND_ADDR, + parse(try_from_str = parse_socket_addr), + )] + pub http_bind_address: SocketAddr, + + /// The address on which IOx will serve Storage gRPC API requests. + #[structopt( + long = "--grpc-bind", + env = "INFLUXDB_IOX_GRPC_BIND_ADDR", + default_value = DEFAULT_GRPC_BIND_ADDR, + parse(try_from_str = parse_socket_addr), + )] + pub grpc_bind_address: SocketAddr, + + /// The location InfluxDB IOx will use to store files locally. + #[structopt(long = "--data-dir", env = "INFLUXDB_IOX_DB_DIR")] + pub database_directory: Option, + + #[structopt( + long = "--object-store", + env = "INFLUXDB_IOX_OBJECT_STORE", + possible_values = &ObjectStore::variants(), + case_insensitive = true, + long_help = r#"Which object storage to use. If not specified, defaults to memory. + +Possible values (case insensitive): + +* memory (default): Effectively no object persistence. +* file: Stores objects in the local filesystem. Must also set `--data-dir`. +* s3: Amazon S3. Must also set `--bucket`, `--aws-access-key-id`, `--aws-secret-access-key`, and + possibly `--aws-default-region`. +* google: Google Cloud Storage. Must also set `--bucket` and `--google-service-account`. +* azure: Microsoft Azure blob storage. Must also set `--bucket`, `--azure-storage-account`, + and `--azure-storage-access-key`. + "#, + )] + pub object_store: Option, + + /// Name of the bucket to use for the object store. Must also set + /// `--object-store` to a cloud object storage to have any effect. + /// + /// If using Google Cloud Storage for the object store, this item as well + /// as `--google-service-account` must be set. + /// + /// If using S3 for the object store, must set this item as well + /// as `--aws-access-key-id` and `--aws-secret-access-key`. Can also set + /// `--aws-default-region` if not using the fallback region. + /// + /// If using Azure for the object store, set this item to the name of a + /// container you've created in the associated storage account, under + /// Blob Service > Containers. Must also set `--azure-storage-account` and + /// `--azure-storage-access-key`. + #[structopt(long = "--bucket", env = "INFLUXDB_IOX_BUCKET")] + pub bucket: Option, + + /// When using Amazon S3 as the object store, set this to an access key that + /// has permission to read from and write to the specified S3 bucket. + /// + /// Must also set `--object-store=s3`, `--bucket`, and + /// `--aws-secret-access-key`. Can also set `--aws-default-region` if not + /// using the fallback region. + /// + /// Prefer the environment variable over the command line flag in shared + /// environments. + #[structopt(long = "--aws-access-key-id", env = "AWS_ACCESS_KEY_ID")] + pub aws_access_key_id: Option, + + /// When using Amazon S3 as the object store, set this to the secret access + /// key that goes with the specified access key ID. + /// + /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`. + /// Can also set `--aws-default-region` if not using the fallback region. + /// + /// Prefer the environment variable over the command line flag in shared + /// environments. + #[structopt(long = "--aws-secret-access-key", env = "AWS_SECRET_ACCESS_KEY")] + pub aws_secret_access_key: Option, + + /// When using Amazon S3 as the object store, set this to the region + /// that goes with the specified bucket if different from the fallback + /// value. + /// + /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`, + /// and `--aws-secret-access-key`. + #[structopt( + long = "--aws-default-region", + env = "AWS_DEFAULT_REGION", + default_value = FALLBACK_AWS_REGION, + )] + pub aws_default_region: String, + + /// When using Google Cloud Storage as the object store, set this to the + /// path to the JSON file that contains the Google credentials. + /// + /// Must also set `--object-store=google` and `--bucket`. + #[structopt(long = "--google-service-account", env = "GOOGLE_SERVICE_ACCOUNT")] + pub google_service_account: Option, + + /// When using Microsoft Azure as the object store, set this to the + /// name you see when going to All Services > Storage accounts > [name]. + /// + /// Must also set `--object-store=azure`, `--bucket`, and + /// `--azure-storage-access-key`. + #[structopt(long = "--azure-storage-account", env = "AZURE_STORAGE_ACCOUNT")] + pub azure_storage_account: Option, + + /// When using Microsoft Azure as the object store, set this to one of the + /// Key values in the Storage account's Settings > Access keys. + /// + /// Must also set `--object-store=azure`, `--bucket`, and + /// `--azure-storage-account`. + /// + /// Prefer the environment variable over the command line flag in shared + /// environments. + #[structopt(long = "--azure-storage-access-key", env = "AZURE_STORAGE_ACCESS_KEY")] + pub azure_storage_access_key: Option, + + /// If set, Jaeger traces are emitted to this host + /// using the OpenTelemetry tracer. + /// + /// NOTE: The OpenTelemetry agent CAN ONLY be + /// configured using environment variables. It CAN NOT be configured + /// using the command line at this time. Some useful variables: + /// + /// * OTEL_SERVICE_NAME: emitter service name (iox by default) + /// * OTEL_EXPORTER_JAEGER_AGENT_HOST: hostname/address of the collector + /// * OTEL_EXPORTER_JAEGER_AGENT_PORT: listening port of the collector. + /// + /// The entire list of variables can be found in + /// https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/sdk-environment-variables.md#jaeger-exporter + #[structopt( + long = "--oetl_exporter_jaeger_agent", + env = "OTEL_EXPORTER_JAEGER_AGENT_HOST" + )] + pub jaeger_host: Option, +} + +pub async fn command(logging_level: LoggingLevel, config: Config) -> Result<()> { + Ok(influxdb_ioxd::main(logging_level, config).await?) +} + +fn parse_socket_addr(s: &str) -> std::io::Result { + let mut addrs = s.to_socket_addrs()?; + // when name resolution fails, to_socket_address returns a validation error + // so generally there is at least one result address, unless the resolver is + // drunk. + Ok(addrs + .next() + .expect("name resolution should return at least one address")) +} + +arg_enum! { + #[derive(Debug, Copy, Clone, PartialEq)] + pub enum ObjectStore { + Memory, + File, + S3, + Google, + Azure, + } +} + +/// How to format output logging messages +#[derive(Debug, Clone, Copy)] +pub enum LogFormat { + /// Default formatted logging + /// + /// Example: + /// ``` + /// level=warn msg="NO PERSISTENCE: using memory for object storage" target="influxdb_iox::influxdb_ioxd" + /// ``` + Rust, + + /// Use the (somwhat pretentiously named) Heroku / logfmt formatted output + /// format + /// + /// Example: + /// ``` + /// Jan 31 13:19:39.059 WARN influxdb_iox::influxdb_ioxd: NO PERSISTENCE: using memory for object storage + /// ``` + LogFmt, +} + +impl Default for LogFormat { + fn default() -> Self { + Self::Rust + } +} + +impl std::str::FromStr for LogFormat { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "rust" => Ok(Self::Rust), + "logfmt" => Ok(Self::LogFmt), + _ => Err(format!( + "Invalid log format '{}'. Valid options: rust, logfmt", + s + )), + } + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; + + fn to_vec(v: &[&str]) -> Vec { + v.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn test_socketaddr() -> Result<(), clap::Error> { + let c = Config::from_iter_safe( + to_vec(&["server", "--api-bind", "127.0.0.1:1234"]).into_iter(), + )?; + assert_eq!( + c.http_bind_address, + SocketAddr::from(([127, 0, 0, 1], 1234)) + ); + + let c = Config::from_iter_safe( + to_vec(&["server", "--api-bind", "localhost:1234"]).into_iter(), + )?; + // depending on where the test runs, localhost will either resolve to a ipv4 or + // an ipv6 addr. + match c.http_bind_address { + SocketAddr::V4(so) => { + assert_eq!(so, SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 1234)) + } + SocketAddr::V6(so) => assert_eq!( + so, + SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 1234, 0, 0) + ), + }; + + assert_eq!( + Config::from_iter_safe( + to_vec(&["server", "--api-bind", "!@INv_a1d(ad0/resp_!"]).into_iter(), + ) + .map_err(|e| e.kind) + .expect_err("must fail"), + clap::ErrorKind::ValueValidation + ); + + Ok(()) + } +} diff --git a/src/commands/server.rs b/src/commands/server.rs index d270bf2969..3bad401af3 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -1,27 +1,12 @@ //! Implementation of command line option for manipulating and showing server //! config -use crate::commands::{logging::LoggingLevel, server_remote}; -use crate::influxdb_ioxd; -use clap::arg_enum; -use std::{net::SocketAddr, net::ToSocketAddrs, path::PathBuf}; +use crate::commands::server_remote; use structopt::StructOpt; use thiserror::Error; -/// The default bind address for the HTTP API. -pub const DEFAULT_API_BIND_ADDR: &str = "127.0.0.1:8080"; - -/// The default bind address for the gRPC. -pub const DEFAULT_GRPC_BIND_ADDR: &str = "127.0.0.1:8082"; - -/// The AWS region to use for Amazon S3 based object storage if none is -/// specified. -pub const FALLBACK_AWS_REGION: &str = "us-east-1"; - #[derive(Debug, Error)] pub enum Error { - #[error("Run: {0}")] - ServerError(#[from] influxdb_ioxd::Error), #[error("Remote: {0}")] RemoteError(#[from] server_remote::Error), } @@ -31,324 +16,11 @@ pub type Result = std::result::Result; #[derive(Debug, StructOpt)] #[structopt(name = "server", about = "IOx server commands")] pub enum Config { - Run(RunConfig), Remote(crate::commands::server_remote::Config), } -#[derive(Debug, StructOpt)] -#[structopt( - name = "run", - about = "Runs in server mode", - long_about = "Run the IOx server.\n\nThe configuration options below can be \ - set either with the command line flags or with the specified environment \ - variable. If there is a file named '.env' in the current working directory, \ - it is sourced before loading the configuration. - -Configuration is loaded from the following sources (highest precedence first): - - command line arguments - - user set environment variables - - .env file contents - - pre-configured default values" -)] -pub struct RunConfig { - /// This controls the IOx server logging level, as described in - /// https://crates.io/crates/env_logger. - /// - /// Levels for different modules can be specified as well. For example - /// `debug,hyper::proto::h1=info` specifies debug logging for all modules - /// except for the `hyper::proto::h1' module which will only display info - /// level logging. - #[structopt(long = "--log", env = "RUST_LOG")] - pub rust_log: Option, - - /// Log message format. Can be one of: - /// - /// "rust" (default) - /// "logfmt" (logfmt/Heroku style - https://brandur.org/logfmt) - #[structopt(long = "--log_format", env = "INFLUXDB_IOX_LOG_FORMAT")] - pub log_format: Option, - - /// This sets logging up with a pre-configured set of convenient log levels. - /// - /// -v means 'info' log levels - /// -vv means 'verbose' log level (with the exception of some particularly - /// low level libraries) - /// - /// This option is ignored if --log / RUST_LOG are set - #[structopt( - short = "-v", - long = "--verbose", - multiple = true, - takes_value = false, - parse(from_occurrences) - )] - pub verbose_count: u64, - - /// The identifier for the server. - /// - /// Used for writing to object storage and as an identifier that is added to - /// replicated writes, WAL segments and Chunks. Must be unique in a group of - /// connected or semi-connected IOx servers. Must be a number that can be - /// represented by a 32-bit unsigned integer. - #[structopt(long = "--writer-id", env = "INFLUXDB_IOX_ID")] - pub writer_id: Option, - - /// The address on which IOx will serve HTTP API requests. - #[structopt( - long = "--api-bind", - env = "INFLUXDB_IOX_BIND_ADDR", - default_value = DEFAULT_API_BIND_ADDR, - parse(try_from_str = parse_socket_addr), - )] - pub http_bind_address: SocketAddr, - - /// The address on which IOx will serve Storage gRPC API requests. - #[structopt( - long = "--grpc-bind", - env = "INFLUXDB_IOX_GRPC_BIND_ADDR", - default_value = DEFAULT_GRPC_BIND_ADDR, - parse(try_from_str = parse_socket_addr), - )] - pub grpc_bind_address: SocketAddr, - - /// The location InfluxDB IOx will use to store files locally. - #[structopt(long = "--data-dir", env = "INFLUXDB_IOX_DB_DIR")] - pub database_directory: Option, - - #[structopt( - long = "--object-store", - env = "INFLUXDB_IOX_OBJECT_STORE", - possible_values = &ObjectStore::variants(), - case_insensitive = true, - long_help = r#"Which object storage to use. If not specified, defaults to memory. - -Possible values (case insensitive): - -* memory (default): Effectively no object persistence. -* file: Stores objects in the local filesystem. Must also set `--data-dir`. -* s3: Amazon S3. Must also set `--bucket`, `--aws-access-key-id`, `--aws-secret-access-key`, and - possibly `--aws-default-region`. -* google: Google Cloud Storage. Must also set `--bucket` and `--google-service-account`. -* azure: Microsoft Azure blob storage. Must also set `--bucket`, `--azure-storage-account`, - and `--azure-storage-access-key`. - "#, - )] - pub object_store: Option, - - /// Name of the bucket to use for the object store. Must also set - /// `--object-store` to a cloud object storage to have any effect. - /// - /// If using Google Cloud Storage for the object store, this item as well - /// as `--google-service-account` must be set. - /// - /// If using S3 for the object store, must set this item as well - /// as `--aws-access-key-id` and `--aws-secret-access-key`. Can also set - /// `--aws-default-region` if not using the fallback region. - /// - /// If using Azure for the object store, set this item to the name of a - /// container you've created in the associated storage account, under - /// Blob Service > Containers. Must also set `--azure-storage-account` and - /// `--azure-storage-access-key`. - #[structopt(long = "--bucket", env = "INFLUXDB_IOX_BUCKET")] - pub bucket: Option, - - /// When using Amazon S3 as the object store, set this to an access key that - /// has permission to read from and write to the specified S3 bucket. - /// - /// Must also set `--object-store=s3`, `--bucket`, and - /// `--aws-secret-access-key`. Can also set `--aws-default-region` if not - /// using the fallback region. - /// - /// Prefer the environment variable over the command line flag in shared - /// environments. - #[structopt(long = "--aws-access-key-id", env = "AWS_ACCESS_KEY_ID")] - pub aws_access_key_id: Option, - - /// When using Amazon S3 as the object store, set this to the secret access - /// key that goes with the specified access key ID. - /// - /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`. - /// Can also set `--aws-default-region` if not using the fallback region. - /// - /// Prefer the environment variable over the command line flag in shared - /// environments. - #[structopt(long = "--aws-secret-access-key", env = "AWS_SECRET_ACCESS_KEY")] - pub aws_secret_access_key: Option, - - /// When using Amazon S3 as the object store, set this to the region - /// that goes with the specified bucket if different from the fallback - /// value. - /// - /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`, - /// and `--aws-secret-access-key`. - #[structopt( - long = "--aws-default-region", - env = "AWS_DEFAULT_REGION", - default_value = FALLBACK_AWS_REGION, - )] - pub aws_default_region: String, - - /// When using Google Cloud Storage as the object store, set this to the - /// path to the JSON file that contains the Google credentials. - /// - /// Must also set `--object-store=google` and `--bucket`. - #[structopt(long = "--google-service-account", env = "GOOGLE_SERVICE_ACCOUNT")] - pub google_service_account: Option, - - /// When using Microsoft Azure as the object store, set this to the - /// name you see when going to All Services > Storage accounts > [name]. - /// - /// Must also set `--object-store=azure`, `--bucket`, and - /// `--azure-storage-access-key`. - #[structopt(long = "--azure-storage-account", env = "AZURE_STORAGE_ACCOUNT")] - pub azure_storage_account: Option, - - /// When using Microsoft Azure as the object store, set this to one of the - /// Key values in the Storage account's Settings > Access keys. - /// - /// Must also set `--object-store=azure`, `--bucket`, and - /// `--azure-storage-account`. - /// - /// Prefer the environment variable over the command line flag in shared - /// environments. - #[structopt(long = "--azure-storage-access-key", env = "AZURE_STORAGE_ACCESS_KEY")] - pub azure_storage_access_key: Option, - - /// If set, Jaeger traces are emitted to this host - /// using the OpenTelemetry tracer. - /// - /// NOTE: The OpenTelemetry agent CAN ONLY be - /// configured using environment variables. It CAN NOT be configured - /// using the command line at this time. Some useful variables: - /// - /// * OTEL_SERVICE_NAME: emitter service name (iox by default) - /// * OTEL_EXPORTER_JAEGER_AGENT_HOST: hostname/address of the collector - /// * OTEL_EXPORTER_JAEGER_AGENT_PORT: listening port of the collector. - /// - /// The entire list of variables can be found in - /// https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/sdk-environment-variables.md#jaeger-exporter - #[structopt( - long = "--oetl_exporter_jaeger_agent", - env = "OTEL_EXPORTER_JAEGER_AGENT_HOST" - )] - pub jaeger_host: Option, -} - -pub async fn command(logging_level: LoggingLevel, url: String, config: Config) -> Result<()> { +pub async fn command(url: String, config: Config) -> Result<()> { match config { - Config::Run(config) => Ok(influxdb_ioxd::main(logging_level, config).await?), Config::Remote(config) => Ok(server_remote::command(url, config).await?), } } - -fn parse_socket_addr(s: &str) -> std::io::Result { - let mut addrs = s.to_socket_addrs()?; - // when name resolution fails, to_socket_address returns a validation error - // so generally there is at least one result address, unless the resolver is - // drunk. - Ok(addrs - .next() - .expect("name resolution should return at least one address")) -} - -arg_enum! { - #[derive(Debug, Copy, Clone, PartialEq)] - pub enum ObjectStore { - Memory, - File, - S3, - Google, - Azure, - } -} - -/// How to format output logging messages -#[derive(Debug, Clone, Copy)] -pub enum LogFormat { - /// Default formatted logging - /// - /// Example: - /// ``` - /// level=warn msg="NO PERSISTENCE: using memory for object storage" target="influxdb_iox::influxdb_ioxd" - /// ``` - Rust, - - /// Use the (somwhat pretentiously named) Heroku / logfmt formatted output - /// format - /// - /// Example: - /// ``` - /// Jan 31 13:19:39.059 WARN influxdb_iox::influxdb_ioxd: NO PERSISTENCE: using memory for object storage - /// ``` - LogFmt, -} - -impl Default for LogFormat { - fn default() -> Self { - Self::Rust - } -} - -impl std::str::FromStr for LogFormat { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "rust" => Ok(Self::Rust), - "logfmt" => Ok(Self::LogFmt), - _ => Err(format!( - "Invalid log format '{}'. Valid options: rust, logfmt", - s - )), - } - } -} - -#[cfg(test)] -mod tests { - - use super::*; - - use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; - - fn to_vec(v: &[&str]) -> Vec { - v.iter().map(|s| s.to_string()).collect() - } - - #[test] - fn test_socketaddr() -> Result<(), clap::Error> { - let c = RunConfig::from_iter_safe( - to_vec(&["server", "--api-bind", "127.0.0.1:1234"]).into_iter(), - )?; - assert_eq!( - c.http_bind_address, - SocketAddr::from(([127, 0, 0, 1], 1234)) - ); - - let c = RunConfig::from_iter_safe( - to_vec(&["server", "--api-bind", "localhost:1234"]).into_iter(), - )?; - // depending on where the test runs, localhost will either resolve to a ipv4 or - // an ipv6 addr. - match c.http_bind_address { - SocketAddr::V4(so) => { - assert_eq!(so, SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 1234)) - } - SocketAddr::V6(so) => assert_eq!( - so, - SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 1234, 0, 0) - ), - }; - - assert_eq!( - RunConfig::from_iter_safe( - to_vec(&["server", "--api-bind", "!@INv_a1d(ad0/resp_!"]).into_iter(), - ) - .map_err(|e| e.kind) - .expect_err("must fail"), - clap::ErrorKind::ValueValidation - ); - - Ok(()) - } -} diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index f8f41b6a60..0ff31ec20c 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -1,6 +1,6 @@ use crate::commands::{ logging::LoggingLevel, - server::{ObjectStore as ObjStoreOpt, RunConfig}, + run::{Config, ObjectStore as ObjStoreOpt}, }; use hyper::Server; use object_store::{ @@ -73,7 +73,7 @@ pub type Result = std::result::Result; /// /// The logging_level passed in is the global setting (e.g. if -v or /// -vv was passed in before 'server') -pub async fn main(logging_level: LoggingLevel, config: RunConfig) -> Result<()> { +pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { // Handle the case if -v/-vv is specified both before and after the server // command let logging_level = logging_level.combine(LoggingLevel::new(config.verbose_count)); @@ -155,10 +155,10 @@ pub async fn main(logging_level: LoggingLevel, config: RunConfig) -> Result<()> Ok(()) } -impl TryFrom<&RunConfig> for ObjectStore { +impl TryFrom<&Config> for ObjectStore { type Error = Error; - fn try_from(config: &RunConfig) -> Result { + fn try_from(config: &Config) -> Result { match config.object_store { Some(ObjStoreOpt::Memory) | None => { Ok(Self::new_in_memory(object_store::memory::InMemory::new())) @@ -285,7 +285,7 @@ mod tests { #[test] fn default_object_store_is_memory() { - let config = RunConfig::from_iter_safe(&["server"]).unwrap(); + let config = Config::from_iter_safe(&["server"]).unwrap(); let object_store = ObjectStore::try_from(&config).unwrap(); @@ -297,7 +297,7 @@ mod tests { #[test] fn explicitly_set_object_store_to_memory() { - let config = RunConfig::from_iter_safe(&["server", "--object-store", "memory"]).unwrap(); + let config = Config::from_iter_safe(&["server", "--object-store", "memory"]).unwrap(); let object_store = ObjectStore::try_from(&config).unwrap(); @@ -309,7 +309,7 @@ mod tests { #[test] fn valid_s3_config() { - let config = RunConfig::from_iter_safe(&[ + let config = Config::from_iter_safe(&[ "server", "--object-store", "s3", @@ -332,7 +332,7 @@ mod tests { #[test] fn s3_config_missing_params() { - let config = RunConfig::from_iter_safe(&["server", "--object-store", "s3"]).unwrap(); + let config = Config::from_iter_safe(&["server", "--object-store", "s3"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); @@ -345,7 +345,7 @@ mod tests { #[test] fn valid_google_config() { - let config = RunConfig::from_iter_safe(&[ + let config = Config::from_iter_safe(&[ "server", "--object-store", "google", @@ -366,7 +366,7 @@ mod tests { #[test] fn google_config_missing_params() { - let config = RunConfig::from_iter_safe(&["server", "--object-store", "google"]).unwrap(); + let config = Config::from_iter_safe(&["server", "--object-store", "google"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); @@ -379,7 +379,7 @@ mod tests { #[test] fn valid_azure_config() { - let config = RunConfig::from_iter_safe(&[ + let config = Config::from_iter_safe(&[ "server", "--object-store", "azure", @@ -402,7 +402,7 @@ mod tests { #[test] fn azure_config_missing_params() { - let config = RunConfig::from_iter_safe(&["server", "--object-store", "azure"]).unwrap(); + let config = Config::from_iter_safe(&["server", "--object-store", "azure"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); @@ -417,7 +417,7 @@ mod tests { fn valid_file_config() { let root = TempDir::new().unwrap(); - let config = RunConfig::from_iter_safe(&[ + let config = Config::from_iter_safe(&[ "server", "--object-store", "file", @@ -436,7 +436,7 @@ mod tests { #[test] fn file_config_missing_params() { - let config = RunConfig::from_iter_safe(&["server", "--object-store", "file"]).unwrap(); + let config = Config::from_iter_safe(&["server", "--object-store", "file"]).unwrap(); let err = ObjectStore::try_from(&config).unwrap_err().to_string(); diff --git a/src/main.rs b/src/main.rs index 3d7e10b201..48ade670df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,7 @@ mod commands { mod input; pub mod logging; pub mod meta; + pub mod run; pub mod server; pub mod server_remote; pub mod stats; @@ -46,13 +47,13 @@ Examples: influxdb_iox # Display all server settings - influxdb_iox server --help + influxdb_iox run --help # Run the InfluxDB IOx server with extra verbose logging - influxdb_iox -v + influxdb_iox run -v # Run InfluxDB IOx with full debug logging specified with RUST_LOG - RUST_LOG=debug influxdb_iox + RUST_LOG=debug influxdb_iox run # converts line protocol formatted data in temperature.lp to out.parquet influxdb_iox convert temperature.lp out.parquet @@ -110,9 +111,10 @@ enum Command { input: String, }, Database(commands::database::Config), - Stats(commands::stats::Config), // Clippy recommended boxing this variant because it's much larger than the others - Server(Box), + Run(Box), + Stats(commands::stats::Config), + Server(commands::server::Config), Writer(commands::writer::Config), } @@ -184,9 +186,16 @@ fn main() -> Result<(), std::io::Error> { } } Command::Server(config) => { + logging_level.setup_basic_logging(); + if let Err(e) = commands::server::command(host, config).await { + eprintln!("Server command failed: {}", e); + std::process::exit(ReturnCode::Failure as _) + } + } + Command::Run(config) => { // Note don't set up basic logging here, different logging rules apply in server // mode - if let Err(e) = commands::server::command(logging_level, host, *config).await { + if let Err(e) = commands::run::command(logging_level, *config).await { eprintln!("Server command failed: {}", e); std::process::exit(ReturnCode::Failure as _) } diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 0b73a2bda2..892c92ca34 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -225,7 +225,6 @@ impl TestServer { let server_process = Command::cargo_bin("influxdb_iox") .unwrap() - .arg("server") .arg("run") // Can enable for debugging //.arg("-vv") @@ -251,7 +250,6 @@ impl TestServer { self.server_process.wait().unwrap(); self.server_process = Command::cargo_bin("influxdb_iox") .unwrap() - .arg("server") .arg("run") // Can enable for debugging //.arg("-vv") From 2604280476319769e9c3bac824367aa59c6c9c78 Mon Sep 17 00:00:00 2001 From: Paul Dix Date: Fri, 12 Mar 2021 13:40:38 -0500 Subject: [PATCH 035/104] refactor: simplify to focus on only sharding writes. routes can come later --- data_types/src/database_rules.rs | 63 +++++++++++++++++--------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index cf692ce4f1..2cc576c697 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -53,11 +53,15 @@ pub struct DatabaseRules { #[serde(default = "MutableBufferConfig::default_option")] pub mutable_buffer_config: Option, - /// An optional config to route writes or queries to other IOx servers. - /// This is useful for sharding a database or copying all or part of - /// requests over to other servers (like shadow production, etc). + /// An optional config to split writes into different "shards". A shard + /// is a logical concept, but the usage is meant to split data into + /// mutually exclusive areas. The rough order of organization is: + /// database -> shard -> partition -> chunk. For example, you could shard + /// based on table name and assign to 1 of 10 shards. Within each + /// shard you would have partitions, which would likely be based off time. + /// This makes it possible to horizontally scale out writes. #[serde(default)] - pub routing_config: Option, + pub shard_config: Option, } impl DatabaseRules { @@ -125,7 +129,7 @@ impl TryFrom for DatabaseRules { partition_template, wal_buffer_config, mutable_buffer_config, - routing_config: None, + shard_config: None, }) } } @@ -709,25 +713,23 @@ impl TryFrom for TemplatePart { } } -/// RoutingConfig defines rules for routing write or query requests to other IOx -/// servers. In the case of writes, routing rules can be used to create copies -/// of the data to get sent out to other servers in a best effort manner. Each -/// route will be checked with a copy of the write or query sent out to each -/// match. +/// ShardConfig defines rules for assigning a line/row to an individual +/// host or a group of hosts. A shard +/// is a logical concept, but the usage is meant to split data into +/// mutually exclusive areas. The rough order of organization is: +/// database -> shard -> partition -> chunk. For example, you could shard +/// based on table name and assign to 1 of 10 shards. Within each +/// shard you would have partitions, which would likely be based off time. +/// This makes it possible to horizontally scale out writes. #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct RoutingConfig { - routes: Vec, -} - -/// A specific named route to match against. Routes can be done based on -/// specific matches to send to a collection of servers or based on hashing to -/// send to a single server in a collection. -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct Route { - /// The name of the route. - pub name: String, +pub struct ShardConfig { /// An optional matcher. If there is a match, the route will be evaluated to - /// the given targets, otherwise the hash ring will be evaluated. + /// the given targets, otherwise the hash ring will be evaluated. This is + /// useful for overriding the hashring function on some hot spot. For + /// example, if you use the table name as the input to the hash function + /// and your ring has 4 slots. If two tables that are very hot get + /// assigned to the same slot you can override that by putting in a + /// specific matcher to pull that table over to a different node. pub specific_targets: Option, /// An optional default hasher which will route to one in a collection of /// nodes. @@ -738,24 +740,27 @@ pub struct Route { pub ignore_errors: bool, } -/// Maps a matcher with specific targets. If it is a match the row/line or query -/// should be sent to all targets. +/// Maps a matcher with specific target group. If the line/row matches +/// it should be sent to the group. #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] pub struct MatcherToTargets { pub matcher: Matcher, - pub targets: Vec, + pub target: NodeGroup, } +/// A collection of IOx nodes +pub type NodeGroup = Vec; + /// HashRing is a rule for creating a hash key for a row and mapping that to /// an individual node on a ring. #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] pub struct HashRing { - /// include the table name in the hash key - pub table_name: Option, + /// If true the table name will be included in the hash key + pub table_name: bool, /// include the values of these columns in the hash key pub columns: Vec, - /// ring of these nodes - pub nodes: Vec, + /// ring of node groups. Each group holds a shard + pub node_groups: Vec, } /// A matcher is used to match routing rules or subscriptions on a row-by-row From 2530be6111cd3cd0f6eabe46a3527e9c0fab81b3 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 14:35:16 -0500 Subject: [PATCH 036/104] docs: Add links to tech talks --- docs/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/README.md b/docs/README.md index a046bf8a60..db26057ef6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,15 @@ This directory contains internal design documentation of potential interest for those who wish to understand how the code works. It is not intended to be general user facing documentation +## IOx Tech Talks + +We hold monthly Tech Talks that explain the project's technical underpinnings, which you can find at the links below: + +* December 2020: Rusty Introduction to Apache Arrow [recording](https://www.youtube.com/watch?v=dQFjKa9vKhM) +* Jan 2021: Data Lifecycle in InfluxDB IOx & How it Uses Object Storage for Persistence [recording](https://www.youtube.com/watch?v=KwdPifHC1Gc) +* February 2021: Intro to the InfluxDB IOx Read Buffer [recording](https://www.youtube.com/watch?v=KslD31VNqPU) [slides](https://www.slideshare.net/influxdata/influxdb-iox-tech-talks-intro-to-the-influxdb-iox-read-buffer-a-readoptimized-inmemory-query-execution-engine) +* March 2021: Query Engine Design and the Rust-Based DataFusion in Apache Arrow[recording](https://www.youtube.com/watch?v=K6eCAVEk4kU) [slides](https://www.slideshare.net/influxdata/influxdb-iox-tech-talks-query-engine-design-and-the-rustbased-datafusion-in-apache-arrow-244161934) + ## Table of Contents: * Rust style and Idiom guide: [style_guide.md](style_guide.md) From a5a7e218402f3c8d9a47494afaf86069f6a5aa9d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 14:36:17 -0500 Subject: [PATCH 037/104] fix: space --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index db26057ef6..21d23e7e43 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,7 @@ We hold monthly Tech Talks that explain the project's technical underpinnings, w * December 2020: Rusty Introduction to Apache Arrow [recording](https://www.youtube.com/watch?v=dQFjKa9vKhM) * Jan 2021: Data Lifecycle in InfluxDB IOx & How it Uses Object Storage for Persistence [recording](https://www.youtube.com/watch?v=KwdPifHC1Gc) * February 2021: Intro to the InfluxDB IOx Read Buffer [recording](https://www.youtube.com/watch?v=KslD31VNqPU) [slides](https://www.slideshare.net/influxdata/influxdb-iox-tech-talks-intro-to-the-influxdb-iox-read-buffer-a-readoptimized-inmemory-query-execution-engine) -* March 2021: Query Engine Design and the Rust-Based DataFusion in Apache Arrow[recording](https://www.youtube.com/watch?v=K6eCAVEk4kU) [slides](https://www.slideshare.net/influxdata/influxdb-iox-tech-talks-query-engine-design-and-the-rustbased-datafusion-in-apache-arrow-244161934) +* March 2021: Query Engine Design and the Rust-Based DataFusion in Apache Arrow [recording](https://www.youtube.com/watch?v=K6eCAVEk4kU) [slides](https://www.slideshare.net/influxdata/influxdb-iox-tech-talks-query-engine-design-and-the-rustbased-datafusion-in-apache-arrow-244161934) ## Table of Contents: From f542d6216e10e8f309aff46e9e4bc272f1d29355 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 14:49:56 -0500 Subject: [PATCH 038/104] fix: Update docs/README.md Co-authored-by: Paul Dix --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 21d23e7e43..c6608339df 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ not intended to be general user facing documentation ## IOx Tech Talks -We hold monthly Tech Talks that explain the project's technical underpinnings, which you can find at the links below: +We hold monthly Tech Talks that explain the project's technical underpinnings. You can register for the [InfluxDB IOx Tech Talks here](https://www.influxdata.com/community-showcase/influxdb-tech-talks/), or you can find links to previous sessions below: * December 2020: Rusty Introduction to Apache Arrow [recording](https://www.youtube.com/watch?v=dQFjKa9vKhM) * Jan 2021: Data Lifecycle in InfluxDB IOx & How it Uses Object Storage for Persistence [recording](https://www.youtube.com/watch?v=KwdPifHC1Gc) From cd66814c3d5347bab1528fe1ed2f54aafe99ad84 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 11 Mar 2021 14:33:04 -0500 Subject: [PATCH 039/104] feat: Add management API and CLI for listing partitions --- generated_types/build.rs | 1 + .../iox/management/v1/partition.proto | 12 ++ .../iox/management/v1/service.proto | 34 +++++ influxdb_iox_client/src/client/management.rs | 74 +++++++++++ src/commands/database.rs | 8 ++ src/commands/database/chunk.rs | 9 +- src/commands/database/partition.rs | 95 +++++++++++++ src/influxdb_ioxd/rpc/management.rs | 46 +++++++ tests/end_to_end_cases/management_api.rs | 99 +++++++++++++- tests/end_to_end_cases/management_cli.rs | 125 ++++++++++++++++-- tests/end_to_end_cases/util.rs | 23 ++++ 11 files changed, 506 insertions(+), 20 deletions(-) create mode 100644 generated_types/protos/influxdata/iox/management/v1/partition.proto create mode 100644 src/commands/database/partition.rs diff --git a/generated_types/build.rs b/generated_types/build.rs index af953aa288..202f9c693f 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -40,6 +40,7 @@ fn generate_grpc_types(root: &Path) -> Result<()> { management_path.join("base_types.proto"), management_path.join("database_rules.proto"), management_path.join("chunk.proto"), + management_path.join("partition.proto"), management_path.join("service.proto"), management_path.join("jobs.proto"), write_path.join("service.proto"), diff --git a/generated_types/protos/influxdata/iox/management/v1/partition.proto b/generated_types/protos/influxdata/iox/management/v1/partition.proto new file mode 100644 index 0000000000..9498cf6f8a --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/partition.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + + +// `Partition` is comprised of data in one or more chunks +// +// TODO: add additional information to this structure (e.g. partition +// names, stats, etc) +message Partition { + // The partitition key of this partition + string key = 1; +} diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index 1a36a8e75a..71a5b81d45 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -4,6 +4,7 @@ package influxdata.iox.management.v1; import "google/longrunning/operations.proto"; import "influxdata/iox/management/v1/database_rules.proto"; import "influxdata/iox/management/v1/chunk.proto"; +import "influxdata/iox/management/v1/partition.proto"; service ManagementService { rpc GetWriterId(GetWriterIdRequest) returns (GetWriterIdResponse); @@ -36,6 +37,13 @@ service ManagementService { metadata_type: "OperationMetadata" }; } + + // List partitions in a database + rpc ListPartitions(ListPartitionsRequest) returns (ListPartitionsResponse); + + // Get detail information about a partition + rpc GetPartition(GetPartitionRequest) returns (GetPartitionResponse); + } message GetWriterIdRequest {} @@ -119,3 +127,29 @@ message DeleteRemoteRequest{ } message DeleteRemoteResponse {} + +// Request to list all partitions from a named database +message ListPartitionsRequest { + // the name of the database + string db_name = 1; +} + +message ListPartitionsResponse { + // All partition keys in a database + repeated string partition_keys = 1; +} + + +// Request to get details of a specific partition from a named database +message GetPartitionRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; +} + +message GetPartitionResponse { + // Detailed information about a partition + Partition partition = 1; +} diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index 10b0c5bd6a..7a14bb42c0 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -104,6 +104,34 @@ pub enum UpdateRemoteError { ServerError(tonic::Status), } +/// Errors returned by Client::list_partitions +#[derive(Debug, Error)] +pub enum ListPartitionsError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::get_partition +#[derive(Debug, Error)] +pub enum GetPartitionError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Partition not found + #[error("Partition not found")] + PartitionNotFound, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// An IOx Management API client. /// /// ```no_run @@ -274,4 +302,50 @@ impl Client { .map_err(UpdateRemoteError::ServerError)?; Ok(()) } + + /// List partition keys of a database + pub async fn list_partitions( + &mut self, + db_name: impl Into, + ) -> Result, ListPartitionsError> { + let db_name = db_name.into(); + let response = self + .inner + .list_partitions(ListPartitionsRequest { db_name }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => ListPartitionsError::DatabaseNotFound, + _ => ListPartitionsError::ServerError(status), + })?; + + let ListPartitionsResponse { partition_keys } = response.into_inner(); + + Ok(partition_keys) + } + + /// Get details about a partition + pub async fn get_partition( + &mut self, + db_name: impl Into, + partition_key: impl Into, + ) -> Result { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + let response = self + .inner + .get_partition(GetPartitionRequest { + db_name, + partition_key, + }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => GetPartitionError::DatabaseNotFound, + _ => GetPartitionError::ServerError(status), + })?; + + let GetPartitionResponse { partition } = response.into_inner(); + + partition.ok_or(GetPartitionError::PartitionNotFound) + } } diff --git a/src/commands/database.rs b/src/commands/database.rs index 208cc4e879..99fb41c791 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -14,6 +14,7 @@ use structopt::StructOpt; use thiserror::Error; mod chunk; +mod partition; #[derive(Debug, Error)] pub enum Error { @@ -46,6 +47,9 @@ pub enum Error { #[error("Error in chunk subcommand: {0}")] Chunk(#[from] chunk::Error), + + #[error("Error in partition subcommand: {0}")] + Partition(#[from] partition::Error), } pub type Result = std::result::Result; @@ -112,6 +116,7 @@ enum Command { Write(Write), Query(Query), Chunk(chunk::Config), + Partition(partition::Config), } pub async fn command(url: String, config: Config) -> Result<()> { @@ -190,6 +195,9 @@ pub async fn command(url: String, config: Config) -> Result<()> { Command::Chunk(config) => { chunk::command(url, config).await?; } + Command::Partition(config) => { + partition::command(url, config).await?; + } } Ok(()) diff --git a/src/commands/database/chunk.rs b/src/commands/database/chunk.rs index b5ab4cf70c..f5974f0385 100644 --- a/src/commands/database/chunk.rs +++ b/src/commands/database/chunk.rs @@ -26,7 +26,7 @@ pub enum Error { pub type Result = std::result::Result; -/// Manage IOx databases +/// Manage IOx chunks #[derive(Debug, StructOpt)] pub struct Config { #[structopt(subcommand)] @@ -55,17 +55,14 @@ pub async fn command(url: String, config: Config) -> Result<()> { let mut client = management::Client::new(connection); - let chunks = client - .list_chunks(db_name) - .await - .map_err(Error::ListChunkError)?; + let chunks = client.list_chunks(db_name).await?; let chunks = chunks .into_iter() .map(ChunkSummary::try_from) .collect::, FieldViolation>>()?; - serde_json::to_writer_pretty(std::io::stdout(), &chunks).map_err(Error::WritingJson)?; + serde_json::to_writer_pretty(std::io::stdout(), &chunks)?; } } diff --git a/src/commands/database/partition.rs b/src/commands/database/partition.rs new file mode 100644 index 0000000000..cec60861af --- /dev/null +++ b/src/commands/database/partition.rs @@ -0,0 +1,95 @@ +//! This module implements the `partition` CLI command +use influxdb_iox_client::{ + connection::Builder, + management::{self, GetPartitionError, ListPartitionsError}, +}; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error listing partitions: {0}")] + ListPartitionsError(#[from] ListPartitionsError), + + #[error("Error getting partition: {0}")] + GetPartitionsError(#[from] GetPartitionError), + + #[error("Error rendering response as JSON: {0}")] + WritingJson(#[from] serde_json::Error), + + // #[error("Error rendering response as JSON: {0}")] + // WritingJson(#[from] serde_json::Error), + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), +} + +pub type Result = std::result::Result; + +/// Manage IOx partitions +#[derive(Debug, StructOpt)] +pub struct Config { + #[structopt(subcommand)] + command: Command, +} + +/// List all known partition keys for a database +#[derive(Debug, StructOpt)] +struct List { + /// The name of the database + db_name: String, +} + +/// Get details of a specific partition in JSON format (TODO) +#[derive(Debug, StructOpt)] +struct Get { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, +} + +/// All possible subcommands for partition +#[derive(Debug, StructOpt)] +enum Command { + // List partitions + List(List), + // Get details about a particular partition + Get(Get), +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + let mut client = management::Client::new(connection); + + match config.command { + Command::List(list) => { + let List { db_name } = list; + let partition_keys = client.list_partitions(db_name).await?; + serde_json::to_writer_pretty(std::io::stdout(), &partition_keys)?; + } + Command::Get(get) => { + let Get { + db_name, + partition_key, + } = get; + + let management::generated_types::Partition { key } = + client.get_partition(db_name, partition_key).await?; + + // TODO: get more details from the partition, andprint it + // out better (i.e. move to using Partition summary that + // is already in data_types) + #[derive(serde::Serialize)] + struct PartitionDetail { + key: String, + } + + let partition_detail = PartitionDetail { key }; + + serde_json::to_writer_pretty(std::io::stdout(), &partition_detail)?; + } + } + + Ok(()) +} diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index fef16299d6..3f171d375e 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -183,6 +183,52 @@ where Ok(Response::new(DeleteRemoteResponse {})) } + + async fn list_partitions( + &self, + request: Request, + ) -> Result, Status> { + let ListPartitionsRequest { db_name } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + let partition_keys = db.partition_keys().map_err(default_db_error_handler)?; + + Ok(Response::new(ListPartitionsResponse { partition_keys })) + } + + async fn get_partition( + &self, + request: Request, + ) -> Result, Status> { + let GetPartitionRequest { + db_name, + partition_key, + } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + // TODO: get more actual partition details + let partition_keys = db.partition_keys().map_err(default_db_error_handler)?; + + let partition = if partition_keys.contains(&partition_key) { + Some(Partition { key: partition_key }) + } else { + None + }; + + Ok(Response::new(GetPartitionResponse { partition })) + } } pub fn make_server( diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 2bc096ace1..268f69a3b8 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -7,7 +7,9 @@ use test_helpers::assert_contains; use crate::common::server_fixture::ServerFixture; -use super::util::{create_readable_database, create_unreadable_database, rand_name}; +use super::util::{ + create_readable_database, create_two_partition_database, create_unreadable_database, rand_name, +}; #[tokio::test] async fn test_list_update_remotes() { @@ -287,3 +289,98 @@ async fn test_chunk_get_errors() { "Cannot read from database: no mutable buffer configured" ); } + +#[tokio::test] +async fn test_partition_list() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = Client::new(fixture.grpc_channel()); + + let db_name = rand_name(); + create_two_partition_database(&db_name, fixture.grpc_channel()).await; + + let mut partitions = management_client + .list_partitions(&db_name) + .await + .expect("listing partition"); + + // ensure the output order is consistent + partitions.sort(); + + let expected = vec!["cpu".to_string(), "mem".to_string()]; + + assert_eq!( + expected, partitions, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, partitions + ); +} + +#[tokio::test] +async fn test_partition_list_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = Client::new(fixture.grpc_channel()); + + let err = management_client + .list_partitions("this database does not exist") + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); +} + +#[tokio::test] +async fn test_partition_get() { + use generated_types::influxdata::iox::management::v1::Partition; + + let fixture = ServerFixture::create_shared().await; + let mut management_client = Client::new(fixture.grpc_channel()); + + let db_name = rand_name(); + create_two_partition_database(&db_name, fixture.grpc_channel()).await; + + let partition_key = "cpu"; + let partition = management_client + .get_partition(&db_name, partition_key) + .await + .expect("getting partition"); + + let expected = Partition { key: "cpu".into() }; + + assert_eq!( + expected, partition, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, partition + ); +} + +#[tokio::test] +async fn test_partition_get_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = Client::new(fixture.grpc_channel()); + let mut write_client = influxdb_iox_client::write::Client::new(fixture.grpc_channel()); + + let err = management_client + .list_partitions("this database does not exist") + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = + vec!["processes,host=foo running=4i,sleeping=514i,total=519i 1591894310000000000"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let err = management_client + .get_partition(&db_name, "non existent partition") + .await + .expect_err("exepcted error getting partition"); + + assert_contains!(err.to_string(), "Partition not found"); +} diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 2d1ba2597f..751a3f0f69 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -96,19 +96,7 @@ async fn test_get_chunks() { "cpu,region=west user=21.0 150", ]; - let lp_data_file = make_temp_file(lp_data.join("\n")); - - Command::cargo_bin("influxdb_iox") - .unwrap() - .arg("database") - .arg("write") - .arg(&db_name) - .arg(lp_data_file.as_ref()) - .arg("--host") - .arg(addr) - .assert() - .success() - .stdout(predicate::str::contains("2 Lines OK")); + load_lp(addr, &db_name, lp_data); let expected = r#"[ { @@ -217,3 +205,114 @@ async fn test_remotes() { .success() .stdout(predicate::str::contains("no remotes configured")); } + +#[tokio::test] +async fn test_list_partitions() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "mem,region=west free=100000 150", + ]; + load_lp(addr, &db_name, lp_data); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("cpu").and(predicate::str::contains("mem"))); +} + +#[tokio::test] +async fn test_list_partitions_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list") + .arg("non_existent_database") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + +#[tokio::test] +async fn test_get_partition() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "mem,region=west free=100000 150", + ]; + load_lp(addr, &db_name, lp_data); + + let expected = r#""key": "cpu""#; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("get") + .arg(&db_name) + .arg("cpu") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +#[tokio::test] +async fn test_get_partition_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("get") + .arg("cpu") + .arg("non_existent_database") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + +/// Loads the specified lines into the named database +fn load_lp(addr: &str, db_name: &str, lp_data: Vec<&str>) { + let lp_data_file = make_temp_file(lp_data.join("\n")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(&db_name) + .arg(lp_data_file.as_ref()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Lines OK")); +} diff --git a/tests/end_to_end_cases/util.rs b/tests/end_to_end_cases/util.rs index a259fc9081..bd12c43fce 100644 --- a/tests/end_to_end_cases/util.rs +++ b/tests/end_to_end_cases/util.rs @@ -59,3 +59,26 @@ pub async fn create_unreadable_database( .await .expect("create database failed"); } + +/// given a channel to talk with the managment api, create a new +/// database with the specified name configured with a 10MB mutable +/// buffer, partitioned on table, with some data written into two partitions +pub async fn create_two_partition_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut write_client = influxdb_iox_client::write::Client::new(channel.clone()); + + let db_name = db_name.into(); + create_readable_database(&db_name, channel).await; + + let lp_lines = vec![ + "mem,host=foo free=27875999744i,cached=0i,available_percent=62.2 1591894320000000000", + "cpu,host=foo running=4i,sleeping=514i,total=519i 1592894310000000000", + ]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); +} From 1b36d6b0cdf218241e1d38ca1dbc014a2e221418 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 15:02:50 -0500 Subject: [PATCH 040/104] fix: Return Partition rather than strings --- .../protos/influxdata/iox/management/v1/service.proto | 4 ++-- influxdb_iox_client/src/client/management.rs | 10 +++++----- src/commands/database/partition.rs | 4 +++- src/influxdb_ioxd/rpc/management.rs | 6 +++++- tests/end_to_end_cases/management_api.rs | 11 +++++++++-- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index 71a5b81d45..bb3a4c18cf 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -135,8 +135,8 @@ message ListPartitionsRequest { } message ListPartitionsResponse { - // All partition keys in a database - repeated string partition_keys = 1; + // All partitions in a database + repeated Partition partitions = 1; } diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index 7a14bb42c0..8b559bde45 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -303,11 +303,11 @@ impl Client { Ok(()) } - /// List partition keys of a database + /// List all partitions of the database pub async fn list_partitions( &mut self, db_name: impl Into, - ) -> Result, ListPartitionsError> { + ) -> Result, ListPartitionsError> { let db_name = db_name.into(); let response = self .inner @@ -318,12 +318,12 @@ impl Client { _ => ListPartitionsError::ServerError(status), })?; - let ListPartitionsResponse { partition_keys } = response.into_inner(); + let ListPartitionsResponse { partitions } = response.into_inner(); - Ok(partition_keys) + Ok(partitions) } - /// Get details about a partition + /// Get details about a specific partition pub async fn get_partition( &mut self, db_name: impl Into, diff --git a/src/commands/database/partition.rs b/src/commands/database/partition.rs index cec60861af..7aed0698ab 100644 --- a/src/commands/database/partition.rs +++ b/src/commands/database/partition.rs @@ -65,7 +65,9 @@ pub async fn command(url: String, config: Config) -> Result<()> { match config.command { Command::List(list) => { let List { db_name } = list; - let partition_keys = client.list_partitions(db_name).await?; + let partitions = client.list_partitions(db_name).await?; + let partition_keys = partitions.into_iter().map(|p| p.key).collect::>(); + serde_json::to_writer_pretty(std::io::stdout(), &partition_keys)?; } Command::Get(get) => { diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index 3f171d375e..0e089d97de 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -198,8 +198,12 @@ where })?; let partition_keys = db.partition_keys().map_err(default_db_error_handler)?; + let partitions = partition_keys + .into_iter() + .map(|key| Partition { key }) + .collect::>(); - Ok(Response::new(ListPartitionsResponse { partition_keys })) + Ok(Response::new(ListPartitionsResponse { partitions })) } async fn get_partition( diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 268f69a3b8..5d3a835dd0 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -304,9 +304,16 @@ async fn test_partition_list() { .expect("listing partition"); // ensure the output order is consistent - partitions.sort(); + partitions.sort_by(|p1, p2| p1.key.cmp(&p2.key)); - let expected = vec!["cpu".to_string(), "mem".to_string()]; + let expected = vec![ + Partition { + key: "cpu".to_string(), + }, + Partition { + key: "mem".to_string(), + }, + ]; assert_eq!( expected, partitions, From cc8390f76340528ca16817934ac6644a709a6e2e Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Fri, 12 Mar 2021 14:25:31 -0500 Subject: [PATCH 041/104] refactor: Use prost's enum helper functions for converting from ints --- src/influxdb_ioxd/rpc/storage/expr.rs | 125 +++++++++----------------- tests/end_to_end_cases/storage_api.rs | 3 +- 2 files changed, 46 insertions(+), 82 deletions(-) diff --git a/src/influxdb_ioxd/rpc/storage/expr.rs b/src/influxdb_ioxd/rpc/storage/expr.rs index eaf0a98720..397254565f 100644 --- a/src/influxdb_ioxd/rpc/storage/expr.rs +++ b/src/influxdb_ioxd/rpc/storage/expr.rs @@ -462,43 +462,30 @@ fn build_node(value: RPCValue, inputs: Vec) -> Result { /// Creates an expr from a "Logical" Node fn build_logical_node(logical: i32, inputs: Vec) -> Result { - // This ideally could be a match, but I couldn't find a safe way - // to match an i32 to RPCLogical except for ths + let logical_enum = RPCLogical::from_i32(logical); - if logical == RPCLogical::And as i32 { - build_binary_expr(Operator::And, inputs) - } else if logical == RPCLogical::Or as i32 { - build_binary_expr(Operator::Or, inputs) - } else { - UnknownLogicalNode { logical }.fail() + match logical_enum { + Some(RPCLogical::And) => build_binary_expr(Operator::And, inputs), + Some(RPCLogical::Or) => build_binary_expr(Operator::Or, inputs), + None => UnknownLogicalNode { logical }.fail(), } } /// Creates an expr from a "Comparsion" Node fn build_comparison_node(comparison: i32, inputs: Vec) -> Result { - // again, this would ideally be a match but I couldn't figure out how to - // match an i32 to the enum values + let comparison_enum = RPCComparison::from_i32(comparison); - if comparison == RPCComparison::Equal as i32 { - build_binary_expr(Operator::Eq, inputs) - } else if comparison == RPCComparison::NotEqual as i32 { - build_binary_expr(Operator::NotEq, inputs) - } else if comparison == RPCComparison::StartsWith as i32 { - StartsWithNotSupported {}.fail() - } else if comparison == RPCComparison::Regex as i32 { - RegExpNotSupported {}.fail() - } else if comparison == RPCComparison::NotRegex as i32 { - NotRegExpNotSupported {}.fail() - } else if comparison == RPCComparison::Lt as i32 { - build_binary_expr(Operator::Lt, inputs) - } else if comparison == RPCComparison::Lte as i32 { - build_binary_expr(Operator::LtEq, inputs) - } else if comparison == RPCComparison::Gt as i32 { - build_binary_expr(Operator::Gt, inputs) - } else if comparison == RPCComparison::Gte as i32 { - build_binary_expr(Operator::GtEq, inputs) - } else { - UnknownComparisonNode { comparison }.fail() + match comparison_enum { + Some(RPCComparison::Equal) => build_binary_expr(Operator::Eq, inputs), + Some(RPCComparison::NotEqual) => build_binary_expr(Operator::NotEq, inputs), + Some(RPCComparison::StartsWith) => StartsWithNotSupported {}.fail(), + Some(RPCComparison::Regex) => RegExpNotSupported {}.fail(), + Some(RPCComparison::NotRegex) => NotRegExpNotSupported {}.fail(), + Some(RPCComparison::Lt) => build_binary_expr(Operator::Lt, inputs), + Some(RPCComparison::Lte) => build_binary_expr(Operator::LtEq, inputs), + Some(RPCComparison::Gt) => build_binary_expr(Operator::Gt, inputs), + Some(RPCComparison::Gte) => build_binary_expr(Operator::GtEq, inputs), + None => UnknownComparisonNode { comparison }.fail(), } } @@ -630,36 +617,23 @@ fn convert_aggregate(aggregate: Option) -> Result }; let aggregate_type = aggregate.r#type; + let aggregate_type_enum = RPCAggregateType::from_i32(aggregate_type); - if aggregate_type == RPCAggregateType::None as i32 { - Ok(QueryAggregate::None) - } else if aggregate_type == RPCAggregateType::Sum as i32 { - Ok(QueryAggregate::Sum) - } else if aggregate_type == RPCAggregateType::Count as i32 { - Ok(QueryAggregate::Count) - } else if aggregate_type == RPCAggregateType::Min as i32 { - Ok(QueryAggregate::Min) - } else if aggregate_type == RPCAggregateType::Max as i32 { - Ok(QueryAggregate::Max) - } else if aggregate_type == RPCAggregateType::First as i32 { - Ok(QueryAggregate::First) - } else if aggregate_type == RPCAggregateType::Last as i32 { - Ok(QueryAggregate::Last) - } else if aggregate_type == RPCAggregateType::Mean as i32 { - Ok(QueryAggregate::Mean) - } else { - UnknownAggregate { aggregate_type }.fail() + match aggregate_type_enum { + Some(RPCAggregateType::None) => Ok(QueryAggregate::None), + Some(RPCAggregateType::Sum) => Ok(QueryAggregate::Sum), + Some(RPCAggregateType::Count) => Ok(QueryAggregate::Count), + Some(RPCAggregateType::Min) => Ok(QueryAggregate::Min), + Some(RPCAggregateType::Max) => Ok(QueryAggregate::Max), + Some(RPCAggregateType::First) => Ok(QueryAggregate::First), + Some(RPCAggregateType::Last) => Ok(QueryAggregate::Last), + Some(RPCAggregateType::Mean) => Ok(QueryAggregate::Mean), + None => UnknownAggregate { aggregate_type }.fail(), } } pub fn convert_group_type(group: i32) -> Result { - if group == RPCGroup::None as i32 { - Ok(RPCGroup::None) - } else if group == RPCGroup::By as i32 { - Ok(RPCGroup::By) - } else { - UnknownGroup { group_type: group }.fail() - } + RPCGroup::from_i32(group).ok_or(Error::UnknownGroup { group_type: group }) } /// Creates a representation of some struct (in another crate that we @@ -774,36 +748,25 @@ fn format_value<'a>(value: &'a RPCValue, f: &mut fmt::Formatter<'_>) -> fmt::Res } fn format_logical(v: i32, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if v == RPCLogical::And as i32 { - write!(f, "AND") - } else if v == RPCLogical::Or as i32 { - write!(f, "Or") - } else { - write!(f, "UNKNOWN_LOGICAL:{}", v) + match RPCLogical::from_i32(v) { + Some(RPCLogical::And) => write!(f, "AND"), + Some(RPCLogical::Or) => write!(f, "Or"), + None => write!(f, "UNKNOWN_LOGICAL:{}", v), } } fn format_comparison(v: i32, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if v == RPCComparison::Equal as i32 { - write!(f, "==") - } else if v == RPCComparison::NotEqual as i32 { - write!(f, "!=") - } else if v == RPCComparison::StartsWith as i32 { - write!(f, "StartsWith") - } else if v == RPCComparison::Regex as i32 { - write!(f, "RegEx") - } else if v == RPCComparison::NotRegex as i32 { - write!(f, "NotRegex") - } else if v == RPCComparison::Lt as i32 { - write!(f, "<") - } else if v == RPCComparison::Lte as i32 { - write!(f, "<=") - } else if v == RPCComparison::Gt as i32 { - write!(f, ">") - } else if v == RPCComparison::Gte as i32 { - write!(f, ">=") - } else { - write!(f, "UNKNOWN_COMPARISON:{}", v) + match RPCComparison::from_i32(v) { + Some(RPCComparison::Equal) => write!(f, "=="), + Some(RPCComparison::NotEqual) => write!(f, "!="), + Some(RPCComparison::StartsWith) => write!(f, "StartsWith"), + Some(RPCComparison::Regex) => write!(f, "RegEx"), + Some(RPCComparison::NotRegex) => write!(f, "NotRegex"), + Some(RPCComparison::Lt) => write!(f, "<"), + Some(RPCComparison::Lte) => write!(f, "<="), + Some(RPCComparison::Gt) => write!(f, ">"), + Some(RPCComparison::Gte) => write!(f, ">="), + None => write!(f, "UNKNOWN_COMPARISON:{}", v), } } diff --git a/tests/end_to_end_cases/storage_api.rs b/tests/end_to_end_cases/storage_api.rs index b547650d71..48da18663d 100644 --- a/tests/end_to_end_cases/storage_api.rs +++ b/tests/end_to_end_cases/storage_api.rs @@ -3,6 +3,7 @@ use futures::prelude::*; use generated_types::{ aggregate::AggregateType, google::protobuf::{Any, Empty}, + measurement_fields_response::FieldType, node::{Comparison, Type as NodeType, Value}, read_group_request::Group, read_response::{frame::Data, *}, @@ -277,7 +278,7 @@ async fn measurement_fields_endpoint( let field = &fields[0]; assert_eq!(field.key, "value"); - assert_eq!(field.r#type, DataType::Float as i32); + assert_eq!(field.r#type(), FieldType::Float); assert_eq!(field.timestamp, scenario.ns_since_epoch() + 4); } From facd02a0ec4a6b20f79a7a850e2dc98ea7cb0aa1 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 12 Mar 2021 16:47:43 -0500 Subject: [PATCH 042/104] feat: configure databases created with CLI with reasonable defaults (#971) * feat: configure databases created with `database create ...` with reasonable defaults * fix: remove truncated comments * fix: Apply suggestions from code review Improve comments Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> * fix: creates a database without a mutable buffer Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- data_types/src/database_rules.rs | 18 +++++- src/commands/database.rs | 47 ++++++++++---- tests/end_to_end_cases/management_cli.rs | 79 +++++++++++++++++++++++- 3 files changed, 129 insertions(+), 15 deletions(-) diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index 2cc576c697..5e5013c952 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -629,10 +629,19 @@ impl TryFrom for PartitionTemplate { /// part of a partition key. #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] pub enum TemplatePart { + /// The name of a table Table, + /// The value in a named column Column(String), + /// Applies a `strftime` format to the "time" column. + /// + /// For example, a time format of "%Y-%m-%d %H:%M:%S" will produce + /// partition key parts such as "2021-03-14 12:25:21" and + /// "2021-04-14 12:24:21" TimeFormat(String), + /// Applies a regex to the value in a string column RegexCapture(RegexCapture), + /// Applies a `strftime` pattern to some column other than "time" StrftimeColumn(StrftimeColumn), } @@ -644,8 +653,15 @@ pub struct RegexCapture { regex: String, } -/// `StrftimeColumn` can be used to create a time based partition key off some +/// [`StrftimeColumn`] is used to create a time based partition key off some /// column other than the builtin `time` column. +/// +/// The value of the named column is formatted using a `strftime` +/// style string. +/// +/// For example, a time format of "%Y-%m-%d %H:%M:%S" will produce +/// partition key parts such as "2021-03-14 12:25:21" and +/// "2021-04-14 12:24:21" #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] pub struct StrftimeColumn { column: String, diff --git a/src/commands/database.rs b/src/commands/database.rs index 99fb41c791..57a8a295e6 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -67,9 +67,10 @@ struct Create { /// The name of the database name: String, - /// Create a mutable buffer of the specified size in bytes - #[structopt(short, long)] - mutable_buffer: Option, + /// Create a mutable buffer of the specified size in bytes. If + /// size is 0, no mutable buffer is created. + #[structopt(short, long, default_value = "104857600")] // 104857600 = 100*1024*1024 + mutable_buffer: u64, } /// Get list of databases @@ -125,18 +126,38 @@ pub async fn command(url: String, config: Config) -> Result<()> { match config.command { Command::Create(command) => { let mut client = management::Client::new(connection); - client - .create_database(DatabaseRules { - name: command.name, - mutable_buffer_config: command.mutable_buffer.map(|buffer_size| { - MutableBufferConfig { - buffer_size, - ..Default::default() - } - }), + + // Configure a mutable buffer if requested + let buffer_size = command.mutable_buffer; + let mutable_buffer_config = if buffer_size > 0 { + Some(MutableBufferConfig { + buffer_size, ..Default::default() }) - .await?; + } else { + None + }; + + let rules = DatabaseRules { + name: command.name, + + mutable_buffer_config, + + // Default to hourly partitions + partition_template: Some(PartitionTemplate { + parts: vec![partition_template::Part { + part: Some(partition_template::part::Part::Time( + "%Y-%m-%d %H:00:00".into(), + )), + }], + }), + + // Note no wal buffer config + ..Default::default() + }; + + client.create_database(rules).await?; + println!("Ok"); } Command::List(_) => { diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 751a3f0f69..ba97a3a568 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -61,6 +61,7 @@ async fn test_create_database() { .success() .stdout(predicate::str::contains("Ok")); + // Listing the databases includes the newly created database Command::cargo_bin("influxdb_iox") .unwrap() .arg("database") @@ -71,6 +72,7 @@ async fn test_create_database() { .success() .stdout(predicate::str::contains(db)); + // Retrieving the database includes the name and a mutable buffer configuration Command::cargo_bin("influxdb_iox") .unwrap() .arg("database") @@ -80,7 +82,82 @@ async fn test_create_database() { .arg(addr) .assert() .success() - .stdout(predicate::str::contains(format!("name: \"{}\"", db))); + .stdout( + predicate::str::contains(db) + .and(predicate::str::contains(format!("name: \"{}\"", db))) + // validate the defaults have been set reasonably + .and(predicate::str::contains("%Y-%m-%d %H:00:00")) + .and(predicate::str::contains("buffer_size: 104857600")) + .and(predicate::str::contains("MutableBufferConfig")), + ); +} + +#[tokio::test] +async fn test_create_database_size() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + let db = &db_name; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db) + .arg("-m") + .arg("1000") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("get") + .arg(db) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout( + predicate::str::contains("buffer_size: 1000") + .and(predicate::str::contains("MutableBufferConfig")), + ); +} + +#[tokio::test] +async fn test_create_database_zero_size() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + let db = &db_name; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db) + .arg("-m") + .arg("0") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("get") + .arg(db) + .arg("--host") + .arg(addr) + .assert() + .success() + // Should not have a mutable buffer + .stdout(predicate::str::contains("MutableBufferConfig").not()); } #[tokio::test] From 3c8a58266a62388dc8a98fd6127b1ec0e4214024 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Fri, 12 Mar 2021 18:19:07 +0100 Subject: [PATCH 043/104] feat: Make S3 auth parameters optional so that if not present, the aws client library can use builtin auth providers, such as the InstanceMetadataProvider, which is commonly used to get the credentials granted to the AWS VM via cloud native mechanism. --- object_store/src/aws.rs | 65 ++++++++++++++++++++++++++--------------- src/influxdb_ioxd.rs | 22 ++++---------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/object_store/src/aws.rs b/object_store/src/aws.rs index 040acdf49d..d9096aa522 100644 --- a/object_store/src/aws.rs +++ b/object_store/src/aws.rs @@ -13,7 +13,7 @@ use futures::{ Stream, StreamExt, TryStreamExt, }; use rusoto_core::ByteStream; -use rusoto_credential::StaticProvider; +use rusoto_credential::{ChainProvider, StaticProvider}; use rusoto_s3::S3; use snafu::{futures::TryStreamExt as _, OptionExt, ResultExt, Snafu}; use std::convert::TryFrom; @@ -108,6 +108,12 @@ pub enum Error { region: String, source: rusoto_core::region::ParseRegionError, }, + + #[snafu(display("Missing aws-access-key"))] + MissingAccessKey, + + #[snafu(display("Missing aws-secret-access-key"))] + MissingSecretAccessKey, } /// Configuration for connecting to [Amazon S3](https://aws.amazon.com/s3/). @@ -285,8 +291,8 @@ impl AmazonS3 { /// Configure a connection to Amazon S3 using the specified credentials in /// the specified Amazon region and bucket pub fn new( - access_key_id: impl Into, - secret_access_key: impl Into, + access_key_id: Option>, + secret_access_key: Option>, region: impl Into, bucket_name: impl Into, ) -> Result { @@ -296,11 +302,22 @@ impl AmazonS3 { let http_client = rusoto_core::request::HttpClient::new() .expect("Current implementation of rusoto_core has no way for this to fail"); - let credentials_provider = - StaticProvider::new_minimal(access_key_id.into(), secret_access_key.into()); + let client = match (access_key_id, secret_access_key) { + (Some(access_key_id), Some(secret_access_key)) => { + let credentials_provider = + StaticProvider::new_minimal(access_key_id.into(), secret_access_key.into()); + rusoto_s3::S3Client::new_with(http_client, credentials_provider, region) + } + (None, Some(_)) => return Err(Error::MissingAccessKey), + (Some(_), None) => return Err(Error::MissingSecretAccessKey), + _ => { + let credentials_provider = ChainProvider::new(); + rusoto_s3::S3Client::new_with(http_client, credentials_provider, region) + } + }; Ok(Self { - client: rusoto_s3::S3Client::new_with(http_client, credentials_provider, region), + client, bucket_name: bucket_name.into(), }) } @@ -502,8 +519,8 @@ mod tests { let config = maybe_skip_integration!(); let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, config.bucket, ) @@ -524,8 +541,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -556,8 +573,8 @@ mod tests { let config = maybe_skip_integration!(); let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -599,8 +616,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -637,8 +654,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -685,8 +702,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -731,8 +748,8 @@ mod tests { let config = maybe_skip_integration!(); let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, config.bucket, ) @@ -757,8 +774,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -795,8 +812,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index 0ff31ec20c..3ffa282aa0 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -197,25 +197,16 @@ impl TryFrom<&Config> for ObjectStore { config.aws_secret_access_key.as_ref(), config.aws_default_region.as_str(), ) { - (Some(bucket), Some(key_id), Some(secret_key), region) => { - Ok(Self::new_amazon_s3( - AmazonS3::new(key_id, secret_key, region, bucket) - .context(InvalidS3Config)?, - )) - } - (bucket, key_id, secret_key, _) => { + (Some(bucket), key_id, secret_key, region) => Ok(Self::new_amazon_s3( + AmazonS3::new(key_id, secret_key, region, bucket) + .context(InvalidS3Config)?, + )), + (bucket, _, _, _) => { let mut missing_args = vec![]; if bucket.is_none() { missing_args.push("bucket"); } - if key_id.is_none() { - missing_args.push("aws-access-key-id"); - } - if secret_key.is_none() { - missing_args.push("aws-secret-access-key"); - } - MissingObjectStoreConfig { object_store: ObjStoreOpt::S3, missing: missing_args.join(", "), @@ -338,8 +329,7 @@ mod tests { assert_eq!( err, - "Specified S3 for the object store, required configuration missing for \ - bucket, aws-access-key-id, aws-secret-access-key" + "Specified S3 for the object store, required configuration missing for bucket" ); } From 91a92e668a5792b189c74936f0e5594f13529ba5 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 15 Mar 2021 09:13:55 -0400 Subject: [PATCH 044/104] refactor: complete migration of end-to-end.rs test to use #[tokio::test] (#987) * refactor: pull Scenario code out of main module * refactor: break out http into tests * refactor: use random org_id and bucket_id * refactor: port read_api to be indepndent * refactor: port last test * refactor: convenience methods to create different clients in end-to-end tests * fix: improve comments Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- tests/common/server_fixture.rs | 18 ++ tests/end-to-end.rs | 286 +------------------- tests/end_to_end_cases/flight_api.rs | 25 +- tests/end_to_end_cases/http.rs | 16 ++ tests/end_to_end_cases/management_api.rs | 32 +-- tests/end_to_end_cases/management_cli.rs | 2 +- tests/end_to_end_cases/mod.rs | 3 +- tests/end_to_end_cases/read_api.rs | 20 +- tests/end_to_end_cases/read_cli.rs | 2 +- tests/end_to_end_cases/scenario.rs | 327 +++++++++++++++++++++++ tests/end_to_end_cases/storage_api.rs | 51 ++-- tests/end_to_end_cases/util.rs | 84 ------ tests/end_to_end_cases/write_api.rs | 6 +- tests/end_to_end_cases/write_cli.rs | 2 +- 14 files changed, 444 insertions(+), 430 deletions(-) create mode 100644 tests/end_to_end_cases/http.rs create mode 100644 tests/end_to_end_cases/scenario.rs delete mode 100644 tests/end_to_end_cases/util.rs diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 892c92ca34..83f157ed7c 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -174,6 +174,24 @@ impl ServerFixture { pub fn influxdb2_client(&self) -> influxdb2_client::Client { influxdb2_client::Client::new(self.http_base(), TOKEN) } + + /// Return a management client suitable for communicating with this + /// server + pub fn management_client(&self) -> influxdb_iox_client::management::Client { + influxdb_iox_client::management::Client::new(self.grpc_channel()) + } + + /// Return a write client suitable for communicating with this + /// server + pub fn write_client(&self) -> influxdb_iox_client::write::Client { + influxdb_iox_client::write::Client::new(self.grpc_channel()) + } + + /// Return a flight client suitable for communicating with this + /// server + pub fn flight_client(&self) -> influxdb_iox_client::flight::Client { + influxdb_iox_client::flight::Client::new(self.grpc_channel()) + } } #[derive(Debug)] diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index c5f0e8f7bc..1f27dbcb09 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -3,291 +3,7 @@ // // The servers under test are managed using [`ServerFixture`] // -// Other rust tests are defined in the various submodules of end_to_end_cases - -use std::convert::TryInto; -use std::str; -use std::time::SystemTime; -use std::u32; - -use futures::prelude::*; -use prost::Message; - -use data_types::{names::org_and_bucket_to_database, DatabaseName}; -use end_to_end_cases::*; -use generated_types::{ - influxdata::iox::management::v1::DatabaseRules, storage_client::StorageClient, ReadSource, - TimestampRange, -}; - -type Error = Box; -type Result = std::result::Result; +// The tests are defined in the submodules of [`end_to_end_cases`] pub mod common; - mod end_to_end_cases; - -use common::server_fixture::*; - -#[tokio::test] -async fn read_and_write_data() { - let fixture = ServerFixture::create_shared().await; - - let influxdb2 = fixture.influxdb2_client(); - let mut storage_client = StorageClient::new(fixture.grpc_channel()); - let mut management_client = - influxdb_iox_client::management::Client::new(fixture.grpc_channel()); - - // These tests share data; TODO: a better way to indicate this - { - let scenario = Scenario::default() - .set_org_id("0000111100001111") - .set_bucket_id("1111000011110000"); - - create_database(&mut management_client, &scenario.database_name()).await; - - let expected_read_data = load_data(&influxdb2, &scenario).await; - let sql_query = "select * from cpu_load_short"; - - read_api::test(&fixture, &scenario, sql_query, &expected_read_data).await; - storage_api::test(&mut storage_client, &scenario).await; - flight_api::test(&fixture, &scenario, sql_query, &expected_read_data).await; - } -} - -// TODO: Randomly generate org and bucket ids to ensure test data independence -// where desired - -#[derive(Debug)] -pub struct Scenario { - org_id_str: String, - bucket_id_str: String, - ns_since_epoch: i64, -} - -impl Default for Scenario { - fn default() -> Self { - let ns_since_epoch = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("System time should have been after the epoch") - .as_nanos() - .try_into() - .expect("Unable to represent system time"); - - Self { - ns_since_epoch, - org_id_str: Default::default(), - bucket_id_str: Default::default(), - } - } -} - -impl Scenario { - fn set_org_id(mut self, org_id: impl Into) -> Self { - self.org_id_str = org_id.into(); - self - } - - fn set_bucket_id(mut self, bucket_id: impl Into) -> Self { - self.bucket_id_str = bucket_id.into(); - self - } - - fn org_id_str(&self) -> &str { - &self.org_id_str - } - - fn bucket_id_str(&self) -> &str { - &self.bucket_id_str - } - - fn org_id(&self) -> u64 { - u64::from_str_radix(&self.org_id_str, 16).unwrap() - } - - fn bucket_id(&self) -> u64 { - u64::from_str_radix(&self.bucket_id_str, 16).unwrap() - } - - fn database_name(&self) -> DatabaseName<'_> { - org_and_bucket_to_database(&self.org_id_str, &self.bucket_id_str).unwrap() - } - - fn ns_since_epoch(&self) -> i64 { - self.ns_since_epoch - } - - fn read_source(&self) -> Option { - let partition_id = u64::from(u32::MAX); - let read_source = ReadSource { - org_id: self.org_id(), - bucket_id: self.bucket_id(), - partition_id, - }; - - let mut d = bytes::BytesMut::new(); - read_source.encode(&mut d).unwrap(); - let read_source = generated_types::google::protobuf::Any { - type_url: "/TODO".to_string(), - value: d.freeze(), - }; - - Some(read_source) - } - - fn timestamp_range(&self) -> Option { - Some(TimestampRange { - start: self.ns_since_epoch(), - end: self.ns_since_epoch() + 10, - }) - } -} - -async fn create_database( - client: &mut influxdb_iox_client::management::Client, - database_name: &str, -) { - client - .create_database(DatabaseRules { - name: database_name.to_string(), - mutable_buffer_config: Some(Default::default()), - ..Default::default() - }) - .await - .unwrap(); -} - -async fn load_data(influxdb2: &influxdb2_client::Client, scenario: &Scenario) -> Vec { - // TODO: make a more extensible way to manage data for tests, such as in - // external fixture files or with factories. - let points = vec![ - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .tag("region", "us-west") - .field("value", 0.64) - .timestamp(scenario.ns_since_epoch()) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .field("value", 27.99) - .timestamp(scenario.ns_since_epoch() + 1) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server02") - .tag("region", "us-west") - .field("value", 3.89) - .timestamp(scenario.ns_since_epoch() + 2) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .tag("region", "us-east") - .field("value", 1234567.891011) - .timestamp(scenario.ns_since_epoch() + 3) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .tag("region", "us-west") - .field("value", 0.000003) - .timestamp(scenario.ns_since_epoch() + 4) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("system") - .tag("host", "server03") - .field("uptime", 1303385) - .timestamp(scenario.ns_since_epoch() + 5) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("swap") - .tag("host", "server01") - .tag("name", "disk0") - .field("in", 3) - .field("out", 4) - .timestamp(scenario.ns_since_epoch() + 6) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("status") - .field("active", true) - .timestamp(scenario.ns_since_epoch() + 7) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("attributes") - .field("color", "blue") - .timestamp(scenario.ns_since_epoch() + 8) - .build() - .unwrap(), - ]; - write_data(&influxdb2, scenario, points).await.unwrap(); - - substitute_nanos( - scenario.ns_since_epoch(), - &[ - "+----------+---------+---------------------+----------------+", - "| host | region | time | value |", - "+----------+---------+---------------------+----------------+", - "| server01 | us-west | ns0 | 0.64 |", - "| server01 | | ns1 | 27.99 |", - "| server02 | us-west | ns2 | 3.89 |", - "| server01 | us-east | ns3 | 1234567.891011 |", - "| server01 | us-west | ns4 | 0.000003 |", - "+----------+---------+---------------------+----------------+", - ], - ) -} - -async fn write_data( - client: &influxdb2_client::Client, - scenario: &Scenario, - points: Vec, -) -> Result<()> { - client - .write( - scenario.org_id_str(), - scenario.bucket_id_str(), - stream::iter(points), - ) - .await?; - Ok(()) -} - -#[tokio::test] -async fn test_http_error_messages() { - let server_fixture = ServerFixture::create_shared().await; - let client = server_fixture.influxdb2_client(); - - // send malformed request (bucket id is invalid) - let result = client - .write_line_protocol("Bar", "Foo", "arbitrary") - .await - .expect_err("Should have errored"); - - let expected_error = "HTTP request returned an error: 400 Bad Request, `{\"error\":\"Error parsing line protocol: A generic parsing error occurred: TakeWhile1\",\"error_code\":100}`"; - assert_eq!(result.to_string(), expected_error); -} - -/// substitutes "ns" --> ns_since_epoch, ns1-->ns_since_epoch+1, etc -fn substitute_nanos(ns_since_epoch: i64, lines: &[&str]) -> Vec { - let substitutions = vec![ - ("ns0", format!("{}", ns_since_epoch)), - ("ns1", format!("{}", ns_since_epoch + 1)), - ("ns2", format!("{}", ns_since_epoch + 2)), - ("ns3", format!("{}", ns_since_epoch + 3)), - ("ns4", format!("{}", ns_since_epoch + 4)), - ("ns5", format!("{}", ns_since_epoch + 5)), - ("ns6", format!("{}", ns_since_epoch + 6)), - ]; - - lines - .iter() - .map(|line| { - let mut line = line.to_string(); - for (from, to) in &substitutions { - line = line.replace(from, to); - } - line - }) - .collect() -} diff --git a/tests/end_to_end_cases/flight_api.rs b/tests/end_to_end_cases/flight_api.rs index 09ecd1ac3d..31ffa9574d 100644 --- a/tests/end_to_end_cases/flight_api.rs +++ b/tests/end_to_end_cases/flight_api.rs @@ -1,14 +1,21 @@ -use crate::{common::server_fixture::ServerFixture, Scenario}; +use super::scenario::Scenario; +use crate::common::server_fixture::ServerFixture; use arrow_deps::assert_table_eq; -use influxdb_iox_client::flight::Client; -pub async fn test( - server_fixture: &ServerFixture, - scenario: &Scenario, - sql_query: &str, - expected_read_data: &[String], -) { - let mut client = Client::new(server_fixture.grpc_channel()); +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_shared().await; + + let influxdb2 = server_fixture.influxdb2_client(); + let mut management_client = server_fixture.management_client(); + + let scenario = Scenario::new(); + scenario.create_database(&mut management_client).await; + + let expected_read_data = scenario.load_data(&influxdb2).await; + let sql_query = "select * from cpu_load_short"; + + let mut client = server_fixture.flight_client(); let mut query_results = client .perform_query(scenario.database_name(), sql_query) diff --git a/tests/end_to_end_cases/http.rs b/tests/end_to_end_cases/http.rs new file mode 100644 index 0000000000..3f076dfc0d --- /dev/null +++ b/tests/end_to_end_cases/http.rs @@ -0,0 +1,16 @@ +use crate::common::server_fixture::ServerFixture; + +#[tokio::test] +async fn test_http_error_messages() { + let server_fixture = ServerFixture::create_shared().await; + let client = server_fixture.influxdb2_client(); + + // send malformed request (bucket id is invalid) + let result = client + .write_line_protocol("Bar", "Foo", "arbitrary") + .await + .expect_err("Should have errored"); + + let expected_error = "HTTP request returned an error: 400 Bad Request, `{\"error\":\"Error parsing line protocol: A generic parsing error occurred: TakeWhile1\",\"error_code\":100}`"; + assert_eq!(result.to_string(), expected_error); +} diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 5d3a835dd0..7be531b09c 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -2,19 +2,19 @@ use std::num::NonZeroU32; use generated_types::google::protobuf::Empty; use generated_types::{google::protobuf::Duration, influxdata::iox::management::v1::*}; -use influxdb_iox_client::management::{Client, CreateDatabaseError}; +use influxdb_iox_client::management::CreateDatabaseError; use test_helpers::assert_contains; use crate::common::server_fixture::ServerFixture; -use super::util::{ +use super::scenario::{ create_readable_database, create_two_partition_database, create_unreadable_database, rand_name, }; #[tokio::test] async fn test_list_update_remotes() { let server_fixture = ServerFixture::create_single_use().await; - let mut client = Client::new(server_fixture.grpc_channel()); + let mut client = server_fixture.management_client(); const TEST_REMOTE_ID_1: u32 = 42; const TEST_REMOTE_ADDR_1: &str = "1.2.3.4:1234"; @@ -74,7 +74,7 @@ async fn test_list_update_remotes() { #[tokio::test] async fn test_set_get_writer_id() { let server_fixture = ServerFixture::create_single_use().await; - let mut client = Client::new(server_fixture.grpc_channel()); + let mut client = server_fixture.management_client(); const TEST_ID: u32 = 42; @@ -91,7 +91,7 @@ async fn test_set_get_writer_id() { #[tokio::test] async fn test_create_database_duplicate_name() { let server_fixture = ServerFixture::create_shared().await; - let mut client = Client::new(server_fixture.grpc_channel()); + let mut client = server_fixture.management_client(); let db_name = rand_name(); @@ -120,7 +120,7 @@ async fn test_create_database_duplicate_name() { #[tokio::test] async fn test_create_database_invalid_name() { let server_fixture = ServerFixture::create_shared().await; - let mut client = Client::new(server_fixture.grpc_channel()); + let mut client = server_fixture.management_client(); let err = client .create_database(DatabaseRules { @@ -136,7 +136,7 @@ async fn test_create_database_invalid_name() { #[tokio::test] async fn test_list_databases() { let server_fixture = ServerFixture::create_shared().await; - let mut client = Client::new(server_fixture.grpc_channel()); + let mut client = server_fixture.management_client(); let name = rand_name(); client @@ -157,7 +157,7 @@ async fn test_list_databases() { #[tokio::test] async fn test_create_get_database() { let server_fixture = ServerFixture::create_shared().await; - let mut client = Client::new(server_fixture.grpc_channel()); + let mut client = server_fixture.management_client(); let db_name = rand_name(); @@ -211,8 +211,8 @@ async fn test_chunk_get() { use generated_types::influxdata::iox::management::v1::{Chunk, ChunkStorage}; let fixture = ServerFixture::create_shared().await; - let mut management_client = Client::new(fixture.grpc_channel()); - let mut write_client = influxdb_iox_client::write::Client::new(fixture.grpc_channel()); + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); let db_name = rand_name(); create_readable_database(&db_name, fixture.grpc_channel()).await; @@ -260,7 +260,7 @@ async fn test_chunk_get() { #[tokio::test] async fn test_chunk_get_errors() { let fixture = ServerFixture::create_shared().await; - let mut management_client = Client::new(fixture.grpc_channel()); + let mut management_client = fixture.management_client(); let db_name = rand_name(); let err = management_client @@ -293,7 +293,7 @@ async fn test_chunk_get_errors() { #[tokio::test] async fn test_partition_list() { let fixture = ServerFixture::create_shared().await; - let mut management_client = Client::new(fixture.grpc_channel()); + let mut management_client = fixture.management_client(); let db_name = rand_name(); create_two_partition_database(&db_name, fixture.grpc_channel()).await; @@ -325,7 +325,7 @@ async fn test_partition_list() { #[tokio::test] async fn test_partition_list_error() { let fixture = ServerFixture::create_shared().await; - let mut management_client = Client::new(fixture.grpc_channel()); + let mut management_client = fixture.management_client(); let err = management_client .list_partitions("this database does not exist") @@ -340,7 +340,7 @@ async fn test_partition_get() { use generated_types::influxdata::iox::management::v1::Partition; let fixture = ServerFixture::create_shared().await; - let mut management_client = Client::new(fixture.grpc_channel()); + let mut management_client = fixture.management_client(); let db_name = rand_name(); create_two_partition_database(&db_name, fixture.grpc_channel()).await; @@ -363,8 +363,8 @@ async fn test_partition_get() { #[tokio::test] async fn test_partition_get_error() { let fixture = ServerFixture::create_shared().await; - let mut management_client = Client::new(fixture.grpc_channel()); - let mut write_client = influxdb_iox_client::write::Client::new(fixture.grpc_channel()); + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); let err = management_client .list_partitions("this database does not exist") diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index ba97a3a568..97c0357d61 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -4,7 +4,7 @@ use test_helpers::make_temp_file; use crate::common::server_fixture::ServerFixture; -use super::util::{create_readable_database, rand_name}; +use super::scenario::{create_readable_database, rand_name}; #[tokio::test] async fn test_writer_id() { diff --git a/tests/end_to_end_cases/mod.rs b/tests/end_to_end_cases/mod.rs index ab69c33893..e21f0dde4f 100644 --- a/tests/end_to_end_cases/mod.rs +++ b/tests/end_to_end_cases/mod.rs @@ -1,9 +1,10 @@ pub mod flight_api; +pub mod http; pub mod management_api; pub mod management_cli; pub mod read_api; pub mod read_cli; +pub mod scenario; pub mod storage_api; -pub mod util; pub mod write_api; pub mod write_cli; diff --git a/tests/end_to_end_cases/read_api.rs b/tests/end_to_end_cases/read_api.rs index b055812ff8..2128782f97 100644 --- a/tests/end_to_end_cases/read_api.rs +++ b/tests/end_to_end_cases/read_api.rs @@ -1,12 +1,18 @@ +use super::scenario::Scenario; use crate::common::server_fixture::ServerFixture; -use crate::Scenario; -pub async fn test( - server_fixture: &ServerFixture, - scenario: &Scenario, - sql_query: &str, - expected_read_data: &[String], -) { +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_shared().await; + let mut management_client = server_fixture.management_client(); + let influxdb2 = server_fixture.influxdb2_client(); + + let scenario = Scenario::new(); + scenario.create_database(&mut management_client).await; + + let expected_read_data = scenario.load_data(&influxdb2).await; + let sql_query = "select * from cpu_load_short"; + let client = reqwest::Client::new(); let db_name = format!("{}_{}", scenario.org_id_str(), scenario.bucket_id_str()); let path = format!("/databases/{}/query", db_name); diff --git a/tests/end_to_end_cases/read_cli.rs b/tests/end_to_end_cases/read_cli.rs index 0c82973f98..739f8ac2de 100644 --- a/tests/end_to_end_cases/read_cli.rs +++ b/tests/end_to_end_cases/read_cli.rs @@ -4,7 +4,7 @@ use test_helpers::make_temp_file; use crate::common::server_fixture::ServerFixture; -use super::util::rand_name; +use super::scenario::rand_name; #[tokio::test] pub async fn test() { diff --git a/tests/end_to_end_cases/scenario.rs b/tests/end_to_end_cases/scenario.rs new file mode 100644 index 0000000000..72c83954eb --- /dev/null +++ b/tests/end_to_end_cases/scenario.rs @@ -0,0 +1,327 @@ +use std::time::SystemTime; + +use generated_types::google::protobuf::Empty; +use generated_types::influxdata::iox::management::v1::*; +use rand::{ + distributions::{Alphanumeric, Standard}, + thread_rng, Rng, +}; + +use std::{convert::TryInto, str, u32}; + +use futures::prelude::*; +use prost::Message; + +use data_types::{names::org_and_bucket_to_database, DatabaseName}; +use generated_types::{influxdata::iox::management::v1::DatabaseRules, ReadSource, TimestampRange}; + +type Error = Box; +type Result = std::result::Result; + +/// A test fixture used for working with the influxdb v2 data model +/// (storage gRPC api and v2 write api). +/// +/// Each scenario is assigned a a random org and bucket id to ensure +/// tests do not interfere with one another +#[derive(Debug)] +pub struct Scenario { + org_id: String, + bucket_id: String, + ns_since_epoch: i64, +} + +impl Scenario { + /// Create a new `Scenario` with a random org_id and bucket_id + pub fn new() -> Self { + let ns_since_epoch = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("System time should have been after the epoch") + .as_nanos() + .try_into() + .expect("Unable to represent system time"); + + Self { + ns_since_epoch, + org_id: rand_id(), + bucket_id: rand_id(), + } + } + + pub fn org_id_str(&self) -> &str { + &self.org_id + } + + pub fn bucket_id_str(&self) -> &str { + &self.bucket_id + } + + pub fn org_id(&self) -> u64 { + u64::from_str_radix(&self.org_id, 16).unwrap() + } + + pub fn bucket_id(&self) -> u64 { + u64::from_str_radix(&self.bucket_id, 16).unwrap() + } + + pub fn database_name(&self) -> DatabaseName<'_> { + org_and_bucket_to_database(&self.org_id, &self.bucket_id).unwrap() + } + + pub fn ns_since_epoch(&self) -> i64 { + self.ns_since_epoch + } + + pub fn read_source(&self) -> Option { + let partition_id = u64::from(u32::MAX); + let read_source = ReadSource { + org_id: self.org_id(), + bucket_id: self.bucket_id(), + partition_id, + }; + + let mut d = bytes::BytesMut::new(); + read_source.encode(&mut d).unwrap(); + let read_source = generated_types::google::protobuf::Any { + type_url: "/TODO".to_string(), + value: d.freeze(), + }; + + Some(read_source) + } + + pub fn timestamp_range(&self) -> Option { + Some(TimestampRange { + start: self.ns_since_epoch(), + end: self.ns_since_epoch() + 10, + }) + } + + /// Create's the database on the server for this scenario + pub async fn create_database(&self, client: &mut influxdb_iox_client::management::Client) { + client + .create_database(DatabaseRules { + name: self.database_name().to_string(), + mutable_buffer_config: Some(Default::default()), + ..Default::default() + }) + .await + .unwrap(); + } + + pub async fn load_data(&self, influxdb2: &influxdb2_client::Client) -> Vec { + // TODO: make a more extensible way to manage data for tests, such as in + // external fixture files or with factories. + let points = vec![ + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .tag("region", "us-west") + .field("value", 0.64) + .timestamp(self.ns_since_epoch()) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .field("value", 27.99) + .timestamp(self.ns_since_epoch() + 1) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server02") + .tag("region", "us-west") + .field("value", 3.89) + .timestamp(self.ns_since_epoch() + 2) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .tag("region", "us-east") + .field("value", 1234567.891011) + .timestamp(self.ns_since_epoch() + 3) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .tag("region", "us-west") + .field("value", 0.000003) + .timestamp(self.ns_since_epoch() + 4) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("system") + .tag("host", "server03") + .field("uptime", 1303385) + .timestamp(self.ns_since_epoch() + 5) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("swap") + .tag("host", "server01") + .tag("name", "disk0") + .field("in", 3) + .field("out", 4) + .timestamp(self.ns_since_epoch() + 6) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("status") + .field("active", true) + .timestamp(self.ns_since_epoch() + 7) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("attributes") + .field("color", "blue") + .timestamp(self.ns_since_epoch() + 8) + .build() + .unwrap(), + ]; + self.write_data(&influxdb2, points).await.unwrap(); + + substitute_nanos( + self.ns_since_epoch(), + &[ + "+----------+---------+---------------------+----------------+", + "| host | region | time | value |", + "+----------+---------+---------------------+----------------+", + "| server01 | us-west | ns0 | 0.64 |", + "| server01 | | ns1 | 27.99 |", + "| server02 | us-west | ns2 | 3.89 |", + "| server01 | us-east | ns3 | 1234567.891011 |", + "| server01 | us-west | ns4 | 0.000003 |", + "+----------+---------+---------------------+----------------+", + ], + ) + } + + async fn write_data( + &self, + client: &influxdb2_client::Client, + points: Vec, + ) -> Result<()> { + client + .write( + self.org_id_str(), + self.bucket_id_str(), + stream::iter(points), + ) + .await?; + Ok(()) + } +} + +/// substitutes "ns" --> ns_since_epoch, ns1-->ns_since_epoch+1, etc +pub fn substitute_nanos(ns_since_epoch: i64, lines: &[&str]) -> Vec { + let substitutions = vec![ + ("ns0", format!("{}", ns_since_epoch)), + ("ns1", format!("{}", ns_since_epoch + 1)), + ("ns2", format!("{}", ns_since_epoch + 2)), + ("ns3", format!("{}", ns_since_epoch + 3)), + ("ns4", format!("{}", ns_since_epoch + 4)), + ("ns5", format!("{}", ns_since_epoch + 5)), + ("ns6", format!("{}", ns_since_epoch + 6)), + ]; + + lines + .iter() + .map(|line| { + let mut line = line.to_string(); + for (from, to) in &substitutions { + line = line.replace(from, to); + } + line + }) + .collect() +} + +/// Return a random string suitable for use as a database name +pub fn rand_name() -> String { + thread_rng() + .sample_iter(&Alphanumeric) + .take(10) + .map(char::from) + .collect() +} + +// return a random 16 digit string comprised of numbers suitable for +// use as a influxdb2 org_id or bucket_id +pub fn rand_id() -> String { + thread_rng() + .sample_iter(&Standard) + .filter_map(|c: u8| { + if c.is_ascii_digit() { + Some(char::from(c)) + } else { + // discard if out of range + None + } + }) + .take(16) + .collect() +} + +/// given a channel to talk with the managment api, create a new +/// database with the specified name configured with a 10MB mutable +/// buffer, partitioned on table +pub async fn create_readable_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + let rules = DatabaseRules { + name: db_name.into(), + partition_template: Some(PartitionTemplate { + parts: vec![partition_template::Part { + part: Some(partition_template::part::Part::Table(Empty {})), + }], + }), + mutable_buffer_config: Some(MutableBufferConfig { + buffer_size: 10 * 1024 * 1024, + ..Default::default() + }), + ..Default::default() + }; + + management_client + .create_database(rules.clone()) + .await + .expect("create database failed"); +} + +/// given a channel to talk with the managment api, create a new +/// database with no mutable buffer configured, no partitioning rules +pub async fn create_unreadable_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + let rules = DatabaseRules { + name: db_name.into(), + ..Default::default() + }; + + management_client + .create_database(rules.clone()) + .await + .expect("create database failed"); +} + +/// given a channel to talk with the managment api, create a new +/// database with the specified name configured with a 10MB mutable +/// buffer, partitioned on table, with some data written into two partitions +pub async fn create_two_partition_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut write_client = influxdb_iox_client::write::Client::new(channel.clone()); + + let db_name = db_name.into(); + create_readable_database(&db_name, channel).await; + + let lp_lines = vec![ + "mem,host=foo free=27875999744i,cached=0i,available_percent=62.2 1591894320000000000", + "cpu,host=foo running=4i,sleeping=514i,total=519i 1592894310000000000", + ]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); +} diff --git a/tests/end_to_end_cases/storage_api.rs b/tests/end_to_end_cases/storage_api.rs index 48da18663d..cd516e73de 100644 --- a/tests/end_to_end_cases/storage_api.rs +++ b/tests/end_to_end_cases/storage_api.rs @@ -1,4 +1,6 @@ -use crate::{common::server_fixture::ServerFixture, create_database, substitute_nanos, Scenario}; +use super::scenario::{substitute_nanos, Scenario}; +use crate::common::server_fixture::ServerFixture; + use futures::prelude::*; use generated_types::{ aggregate::AggregateType, @@ -12,20 +14,30 @@ use generated_types::{ MeasurementTagValuesRequest, Node, Predicate, ReadFilterRequest, ReadGroupRequest, ReadWindowAggregateRequest, Tag, TagKeysRequest, TagValuesRequest, TimestampRange, }; -use influxdb_iox_client::management; use std::str; use test_helpers::tag_key_bytes_to_strings; use tonic::transport::Channel; -pub async fn test(storage_client: &mut StorageClient, scenario: &Scenario) { - capabilities_endpoint(storage_client).await; - read_filter_endpoint(storage_client, scenario).await; - tag_keys_endpoint(storage_client, scenario).await; - tag_values_endpoint(storage_client, scenario).await; - measurement_names_endpoint(storage_client, scenario).await; - measurement_tag_keys_endpoint(storage_client, scenario).await; - measurement_tag_values_endpoint(storage_client, scenario).await; - measurement_fields_endpoint(storage_client, scenario).await; +#[tokio::test] +pub async fn test() { + let storage_fixture = ServerFixture::create_shared().await; + + let influxdb2 = storage_fixture.influxdb2_client(); + let mut storage_client = StorageClient::new(storage_fixture.grpc_channel()); + let mut management_client = storage_fixture.management_client(); + + let scenario = Scenario::new(); + scenario.create_database(&mut management_client).await; + scenario.load_data(&influxdb2).await; + + capabilities_endpoint(&mut storage_client).await; + read_filter_endpoint(&mut storage_client, &scenario).await; + tag_keys_endpoint(&mut storage_client, &scenario).await; + tag_values_endpoint(&mut storage_client, &scenario).await; + measurement_names_endpoint(&mut storage_client, &scenario).await; + measurement_tag_keys_endpoint(&mut storage_client, &scenario).await; + measurement_tag_values_endpoint(&mut storage_client, &scenario).await; + measurement_fields_endpoint(&mut storage_client, &scenario).await; } /// Validate that capabilities storage endpoint is hooked up @@ -285,15 +297,12 @@ async fn measurement_fields_endpoint( #[tokio::test] pub async fn read_group_test() { let fixture = ServerFixture::create_shared().await; - let mut management = management::Client::new(fixture.grpc_channel()); + let mut management = fixture.management_client(); let mut storage_client = StorageClient::new(fixture.grpc_channel()); let influxdb2 = fixture.influxdb2_client(); - let scenario = Scenario::default() - .set_org_id("0000111100001110") - .set_bucket_id("1111000011110001"); - - create_database(&mut management, &scenario.database_name()).await; + let scenario = Scenario::new(); + scenario.create_database(&mut management).await; load_read_group_data(&influxdb2, &scenario).await; @@ -540,16 +549,14 @@ async fn test_read_group_last_agg( #[tokio::test] pub async fn read_window_aggregate_test() { let fixture = ServerFixture::create_shared().await; - let mut management = management::Client::new(fixture.grpc_channel()); + let mut management = fixture.management_client(); let mut storage_client = StorageClient::new(fixture.grpc_channel()); let influxdb2 = fixture.influxdb2_client(); - let scenario = Scenario::default() - .set_org_id("0000111100001100") - .set_bucket_id("1111000011110011"); + let scenario = Scenario::new(); let read_source = scenario.read_source(); - create_database(&mut management, &scenario.database_name()).await; + scenario.create_database(&mut management).await; let line_protocol = vec![ "h2o,state=MA,city=Boston temp=70.0 100", diff --git a/tests/end_to_end_cases/util.rs b/tests/end_to_end_cases/util.rs deleted file mode 100644 index bd12c43fce..0000000000 --- a/tests/end_to_end_cases/util.rs +++ /dev/null @@ -1,84 +0,0 @@ -use rand::{distributions::Alphanumeric, thread_rng, Rng}; - -use generated_types::google::protobuf::Empty; -use generated_types::influxdata::iox::management::v1::*; - -/// Return a random string suitable for use as a database name -pub fn rand_name() -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .take(10) - .map(char::from) - .collect() -} - -/// given a channel to talk with the managment api, create a new -/// database with the specified name configured with a 10MB mutable -/// buffer, partitioned on table -pub async fn create_readable_database( - db_name: impl Into, - channel: tonic::transport::Channel, -) { - let mut management_client = influxdb_iox_client::management::Client::new(channel); - - let rules = DatabaseRules { - name: db_name.into(), - partition_template: Some(PartitionTemplate { - parts: vec![partition_template::Part { - part: Some(partition_template::part::Part::Table(Empty {})), - }], - }), - mutable_buffer_config: Some(MutableBufferConfig { - buffer_size: 10 * 1024 * 1024, - ..Default::default() - }), - ..Default::default() - }; - - management_client - .create_database(rules.clone()) - .await - .expect("create database failed"); -} - -/// given a channel to talk with the managment api, create a new -/// database with no mutable buffer configured, no partitioning rules -pub async fn create_unreadable_database( - db_name: impl Into, - channel: tonic::transport::Channel, -) { - let mut management_client = influxdb_iox_client::management::Client::new(channel); - - let rules = DatabaseRules { - name: db_name.into(), - ..Default::default() - }; - - management_client - .create_database(rules.clone()) - .await - .expect("create database failed"); -} - -/// given a channel to talk with the managment api, create a new -/// database with the specified name configured with a 10MB mutable -/// buffer, partitioned on table, with some data written into two partitions -pub async fn create_two_partition_database( - db_name: impl Into, - channel: tonic::transport::Channel, -) { - let mut write_client = influxdb_iox_client::write::Client::new(channel.clone()); - - let db_name = db_name.into(); - create_readable_database(&db_name, channel).await; - - let lp_lines = vec![ - "mem,host=foo free=27875999744i,cached=0i,available_percent=62.2 1591894320000000000", - "cpu,host=foo running=4i,sleeping=514i,total=519i 1592894310000000000", - ]; - - write_client - .write(&db_name, lp_lines.join("\n")) - .await - .expect("write succeded"); -} diff --git a/tests/end_to_end_cases/write_api.rs b/tests/end_to_end_cases/write_api.rs index a1a67aa283..3ab14ef312 100644 --- a/tests/end_to_end_cases/write_api.rs +++ b/tests/end_to_end_cases/write_api.rs @@ -1,14 +1,14 @@ -use influxdb_iox_client::write::{self, WriteError}; +use influxdb_iox_client::write::WriteError; use test_helpers::assert_contains; use crate::common::server_fixture::ServerFixture; -use super::util::{create_readable_database, rand_name}; +use super::scenario::{create_readable_database, rand_name}; #[tokio::test] async fn test_write() { let fixture = ServerFixture::create_shared().await; - let mut write_client = write::Client::new(fixture.grpc_channel()); + let mut write_client = fixture.write_client(); let db_name = rand_name(); create_readable_database(&db_name, fixture.grpc_channel()).await; diff --git a/tests/end_to_end_cases/write_cli.rs b/tests/end_to_end_cases/write_cli.rs index b3903c831f..e4adf7e04b 100644 --- a/tests/end_to_end_cases/write_cli.rs +++ b/tests/end_to_end_cases/write_cli.rs @@ -4,7 +4,7 @@ use test_helpers::make_temp_file; use crate::common::server_fixture::ServerFixture; -use super::util::rand_name; +use super::scenario::rand_name; #[tokio::test] async fn test() { From 8bd7a39607e4c381cf4337683bafe76f8fcca18c Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 15 Mar 2021 14:40:07 +0100 Subject: [PATCH 045/104] fix: Use InstanceMetadataProvider directly Rusoto's ChainProvider swallows the error message produced by the underlying InstanceMetadataProvider (see https://github.com/rusoto/rusoto/blob/d59d716f097f64a713b3bbcd5fc7b84e210647e4/rusoto/credential/src/lib.rs#L397) making it hard for us to know why it's not working in our staging cluster. --- object_store/src/aws.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/object_store/src/aws.rs b/object_store/src/aws.rs index d9096aa522..16a6cb47af 100644 --- a/object_store/src/aws.rs +++ b/object_store/src/aws.rs @@ -13,7 +13,7 @@ use futures::{ Stream, StreamExt, TryStreamExt, }; use rusoto_core::ByteStream; -use rusoto_credential::{ChainProvider, StaticProvider}; +use rusoto_credential::{InstanceMetadataProvider, StaticProvider}; use rusoto_s3::S3; use snafu::{futures::TryStreamExt as _, OptionExt, ResultExt, Snafu}; use std::convert::TryFrom; @@ -311,7 +311,7 @@ impl AmazonS3 { (None, Some(_)) => return Err(Error::MissingAccessKey), (Some(_), None) => return Err(Error::MissingSecretAccessKey), _ => { - let credentials_provider = ChainProvider::new(); + let credentials_provider = InstanceMetadataProvider::new(); rusoto_s3::S3Client::new_with(http_client, credentials_provider, region) } }; From 65f7a1ac5b2b8b20ccaa128821c9b59222e7ce73 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Mon, 15 Mar 2021 15:42:19 +0000 Subject: [PATCH 046/104] fix: use consistent crate versions (#989) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- Cargo.lock | 162 ++++++-------------------- Cargo.toml | 4 +- benches/encoders.rs | 4 +- benches/packers.rs | 2 +- data_types/Cargo.toml | 2 +- generated_types/Cargo.toml | 2 +- influxdb_iox_client/Cargo.toml | 2 +- influxdb_tsm/Cargo.toml | 2 +- influxdb_tsm/src/encoders/simple8b.rs | 2 +- mem_qe/Cargo.toml | 4 +- mutable_buffer/Cargo.toml | 2 +- packers/Cargo.toml | 2 +- packers/src/sorter.rs | 4 +- query/Cargo.toml | 2 +- read_buffer/Cargo.toml | 4 +- read_buffer/benches/dictionary.rs | 11 +- read_buffer/benches/row_group.rs | 29 +++-- server/Cargo.toml | 2 +- 18 files changed, 79 insertions(+), 163 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fd205a112..9b1893c62e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,7 +324,7 @@ dependencies = [ "cexpr", "clang-sys", "clap", - "env_logger 0.8.3", + "env_logger", "lazy_static", "lazycell", "log", @@ -635,26 +635,16 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +checksum = "fd01a6eb3daaafa260f6fc94c3a6c36390abc2080e38e3e34ced87393fb77d80" dependencies = [ - "cfg-if 0.1.10", - "crossbeam-channel 0.4.4", - "crossbeam-deque 0.7.3", - "crossbeam-epoch 0.8.2", + "cfg-if 1.0.0", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", "crossbeam-queue", - "crossbeam-utils 0.7.2", -] - -[[package]] -name = "crossbeam-channel" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" -dependencies = [ - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] @@ -664,18 +654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] @@ -685,23 +664,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.3", - "crossbeam-utils 0.8.3", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] @@ -711,32 +675,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", + "crossbeam-utils", "lazy_static", - "memoffset 0.6.1", + "memoffset", "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.2.3" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "cfg-if 1.0.0", + "crossbeam-utils", ] [[package]] @@ -829,7 +781,7 @@ dependencies = [ "parquet", "paste", "pin-project-lite", - "sqlparser 0.8.0", + "sqlparser", "tokio", "tokio-stream", "unicode-segmentation", @@ -947,19 +899,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.8.3" @@ -967,7 +906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "atty", - "humantime 2.1.0", + "humantime", "log", "regex", "termcolor", @@ -1367,15 +1306,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "humantime" version = "2.1.0" @@ -1486,7 +1416,7 @@ dependencies = [ "data_types", "dirs 3.0.1", "dotenv", - "env_logger 0.7.1", + "env_logger", "flate2", "futures", "generated_types", @@ -1512,7 +1442,7 @@ dependencies = [ "prettytable-rs", "prost", "query", - "rand 0.7.3", + "rand 0.8.3", "read_buffer", "reqwest", "routerify", @@ -1572,7 +1502,7 @@ dependencies = [ "flate2", "hex", "integer-encoding", - "rand 0.7.3", + "rand 0.8.3", "snafu", "snap", "test_helpers", @@ -1789,12 +1719,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md5" version = "0.7.0" @@ -1810,7 +1734,7 @@ dependencies = [ "criterion", "croaring", "crossbeam", - "env_logger 0.7.1", + "env_logger", "human_format", "packers", "snafu", @@ -1823,15 +1747,6 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.1" @@ -2237,7 +2152,7 @@ dependencies = [ "data_types", "human_format", "influxdb_tsm", - "rand 0.7.3", + "rand 0.8.3", "snafu", "test_helpers", "tracing", @@ -2592,7 +2507,7 @@ dependencies = [ "influxdb_line_protocol", "parking_lot", "snafu", - "sqlparser 0.6.1", + "sqlparser", "test_helpers", "tokio", "tokio-stream", @@ -2679,12 +2594,12 @@ dependencies = [ [[package]] name = "rand_distr" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9532ada3929fb8b2e9dbe28d1e06c9b2cc65813f074fcb6bd5fbefeff9d56" +checksum = "da9e8f32ad24fb80d07d2323a9a2ce8b30d68a62b8cb4df88119ff49a698f038" dependencies = [ "num-traits", - "rand 0.7.3", + "rand 0.8.3", ] [[package]] @@ -2712,7 +2627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" dependencies = [ "autocfg", - "crossbeam-deque 0.8.0", + "crossbeam-deque", "either", "rayon-core", ] @@ -2723,9 +2638,9 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ - "crossbeam-channel 0.5.0", - "crossbeam-deque 0.8.0", - "crossbeam-utils 0.8.3", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", "lazy_static", "num_cpus", ] @@ -2743,7 +2658,7 @@ dependencies = [ "itertools 0.9.0", "packers", "permutation", - "rand 0.7.3", + "rand 0.8.3", "rand_distr", "snafu", ] @@ -2978,7 +2893,7 @@ dependencies = [ "base64 0.13.0", "blake2b_simd", "constant_time_eq", - "crossbeam-utils 0.8.3", + "crossbeam-utils", ] [[package]] @@ -3341,15 +3256,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "sqlparser" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fa7478852b3ea28f0d21a42b2d7dade24ba4aa72e22bf66982e4b587a7f608" -dependencies = [ - "log", -] - [[package]] name = "sqlparser" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index efd5b79e1e..8d339c813d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,7 @@ clap = "2.33.1" csv = "1.1" dirs = "3.0.1" dotenv = "0.15.0" -env_logger = "0.7.1" +env_logger = "0.8.3" flate2 = "1.0" futures = "0.3.1" http = "0.2.0" @@ -104,7 +104,7 @@ criterion = "0.3" flate2 = "1.0" hex = "0.4.2" predicates = "1.0.4" -rand = "0.7.2" +rand = "0.8.3" reqwest = "0.11" tempfile = "3.1.0" diff --git a/benches/encoders.rs b/benches/encoders.rs index 34f2ea1ede..602fdb8ad0 100644 --- a/benches/encoders.rs +++ b/benches/encoders.rs @@ -204,7 +204,7 @@ fn integer_encode_random(c: &mut Criterion) { &LARGER_BATCH_SIZES, |batch_size| { (1..batch_size) - .map(|_| rand::thread_rng().gen_range(0, 100)) + .map(|_| rand::thread_rng().gen_range(0..100)) .collect() }, influxdb_tsm::encoders::integer::encode, @@ -323,7 +323,7 @@ fn integer_decode_random(c: &mut Criterion) { &LARGER_BATCH_SIZES, |batch_size| { let decoded: Vec = (1..batch_size) - .map(|_| rand::thread_rng().gen_range(0, 100)) + .map(|_| rand::thread_rng().gen_range(0..100)) .collect(); let mut encoded = vec![]; influxdb_tsm::encoders::integer::encode(&decoded, &mut encoded).unwrap(); diff --git a/benches/packers.rs b/benches/packers.rs index b0baadcc82..a34a66958c 100644 --- a/benches/packers.rs +++ b/benches/packers.rs @@ -52,7 +52,7 @@ fn i64_vec_with_nulls(size: usize, null_percent: usize) -> Vec> { let mut a = Vec::with_capacity(size); // insert 10% null values for _ in 0..size { - if rng.gen_range(0, null_percent) == 0 { + if rng.gen_range(0..null_percent) == 0 { a.push(None); } else { a.push(Some(1_u64)); diff --git a/data_types/Cargo.toml b/data_types/Cargo.toml index d18b0fc2bd..b2c669ae26 100644 --- a/data_types/Cargo.toml +++ b/data_types/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" arrow_deps = { path = "../arrow_deps" } chrono = { version = "0.4", features = ["serde"] } crc32fast = "1.2.0" -flatbuffers = "0.6" +flatbuffers = "0.6" # TODO: Update to 0.8 generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } percent-encoding = "2.1.0" diff --git a/generated_types/Cargo.toml b/generated_types/Cargo.toml index 3526ed5276..7a245194cc 100644 --- a/generated_types/Cargo.toml +++ b/generated_types/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Dix "] edition = "2018" [dependencies] # In alphabetical order -flatbuffers = "0.6.1" +flatbuffers = "0.6" # TODO: Update to 0.8 futures = "0.3.1" prost = "0.7" prost-types = "0.7" diff --git a/influxdb_iox_client/Cargo.toml b/influxdb_iox_client/Cargo.toml index 67e6274b8b..1dc3fb8d34 100644 --- a/influxdb_iox_client/Cargo.toml +++ b/influxdb_iox_client/Cargo.toml @@ -23,5 +23,5 @@ tokio = { version = "1.0", features = ["macros"] } tonic = { version = "0.4.0" } [dev-dependencies] # In alphabetical order -rand = "0.8.1" +rand = "0.8.3" serde_json = "1.0" diff --git a/influxdb_tsm/Cargo.toml b/influxdb_tsm/Cargo.toml index bfe4a7251b..1355210456 100644 --- a/influxdb_tsm/Cargo.toml +++ b/influxdb_tsm/Cargo.toml @@ -13,5 +13,5 @@ tracing = "0.1" [dev-dependencies] # In alphabetical order flate2 = "1.0" hex = "0.4.2" -rand = "0.7.2" +rand = "0.8.3" test_helpers = { path = "../test_helpers" } diff --git a/influxdb_tsm/src/encoders/simple8b.rs b/influxdb_tsm/src/encoders/simple8b.rs index b6e1297fe5..131050923f 100644 --- a/influxdb_tsm/src/encoders/simple8b.rs +++ b/influxdb_tsm/src/encoders/simple8b.rs @@ -381,7 +381,7 @@ mod tests { let mut a = Vec::with_capacity(n as usize); for i in 0..n { let top_bit = (i & 1) << (bits - 1); - let v = rng.gen_range(0, max) | top_bit; + let v = rng.gen_range(0..max) | top_bit; assert!(v < max); a.push(v); } diff --git a/mem_qe/Cargo.toml b/mem_qe/Cargo.toml index dc36751890..23ff477433 100644 --- a/mem_qe/Cargo.toml +++ b/mem_qe/Cargo.toml @@ -8,8 +8,8 @@ edition = "2018" arrow_deps = { path = "../arrow_deps" } chrono = "0.4" croaring = "0.4.5" -crossbeam = "0.7.3" -env_logger = "0.7.1" +crossbeam = "0.8" +env_logger = "0.8.3" human_format = "1.0.3" packers = { path = "../packers" } snafu = "0.6.8" diff --git a/mutable_buffer/Cargo.toml b/mutable_buffer/Cargo.toml index 46bb17eeb3..638a62c2f9 100644 --- a/mutable_buffer/Cargo.toml +++ b/mutable_buffer/Cargo.toml @@ -18,7 +18,7 @@ arrow_deps = { path = "../arrow_deps" } async-trait = "0.1" chrono = "0.4" data_types = { path = "../data_types" } -flatbuffers = "0.6.1" +flatbuffers = "0.6" # TODO: Update to 0.8 generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } snafu = "0.6.2" diff --git a/packers/Cargo.toml b/packers/Cargo.toml index 94a737d040..fa9b92124c 100644 --- a/packers/Cargo.toml +++ b/packers/Cargo.toml @@ -13,5 +13,5 @@ snafu = "0.6.2" tracing = "0.1" [dev-dependencies] # In alphabetical order -rand = "0.7.3" +rand = "0.8.3" test_helpers = { path = "../test_helpers" } diff --git a/packers/src/sorter.rs b/packers/src/sorter.rs index 6b6bf49f7c..016a72f67f 100644 --- a/packers/src/sorter.rs +++ b/packers/src/sorter.rs @@ -387,7 +387,7 @@ mod test { for _ in 0..250 { let packer: Packer = Packer::from( (0..1000) - .map(|_| rng.gen_range(0, 20)) + .map(|_| rng.gen_range(0..20)) .collect::>(), ); let mut packers = vec![Packers::Integer(packer)]; @@ -410,7 +410,7 @@ mod test { for _ in 0..250 { let packer: Packer = Packer::from( (0..1000) - .map(|_| format!("{:?}", rng.gen_range(0, 20))) + .map(|_| format!("{:?}", rng.gen_range(0..20))) .collect::>(), ); let mut packers = vec![Packers::String(packer)]; diff --git a/query/Cargo.toml b/query/Cargo.toml index ce25b22fb2..2273117036 100644 --- a/query/Cargo.toml +++ b/query/Cargo.toml @@ -23,7 +23,7 @@ futures = "0.3.7" influxdb_line_protocol = { path = "../influxdb_line_protocol" } parking_lot = "0.11.1" snafu = "0.6.2" -sqlparser = "0.6.1" +sqlparser = "0.8.0" tokio = { version = "1.0", features = ["macros"] } tokio-stream = "0.1.2" tracing = "0.1" diff --git a/read_buffer/Cargo.toml b/read_buffer/Cargo.toml index 7e5a52332d..de9018c428 100644 --- a/read_buffer/Cargo.toml +++ b/read_buffer/Cargo.toml @@ -23,8 +23,8 @@ snafu = "0.6" [dev-dependencies] # In alphabetical order criterion = "0.3.3" -rand = "0.7.3" -rand_distr = "0.3.0" +rand = "0.8.3" +rand_distr = "0.4.0" [[bench]] name = "database" diff --git a/read_buffer/benches/dictionary.rs b/read_buffer/benches/dictionary.rs index bb3aff61bc..6558bd0a2b 100644 --- a/read_buffer/benches/dictionary.rs +++ b/read_buffer/benches/dictionary.rs @@ -68,16 +68,16 @@ fn benchmark_select( let value = match location { Location::Start => { // find a value in the column close to the beginning. - &col_data[rng.gen_range(0, col_data.len() / 20)] // something in first 5% + &col_data[rng.gen_range(0..col_data.len() / 20)] // something in first 5% } Location::Middle => { // find a value in the column somewhere in the middle let fifth = col_data.len() / 5; - &col_data[rng.gen_range(2 * fifth, 3 * fifth)] // something in middle fifth + &col_data[rng.gen_range(2 * fifth..3 * fifth)] // something in middle fifth } Location::End => { &col_data - [rng.gen_range(col_data.len() - (col_data.len() / 9), col_data.len())] + [rng.gen_range(col_data.len() - (col_data.len() / 9)..col_data.len())] } // something in the last ~10% }; @@ -139,7 +139,10 @@ fn generate_column(rows: usize, rows_per_value: usize, rng: &mut ThreadRng) -> V for _ in 0..distinct_values { let value = format!( "value-{}", - rng.sample_iter(&Alphanumeric).take(8).collect::() + rng.sample_iter(&Alphanumeric) + .map(char::from) + .take(8) + .collect::() ); col.extend(std::iter::repeat(value).take(rows_per_value)); } diff --git a/read_buffer/benches/row_group.rs b/read_buffer/benches/row_group.rs index c8c141aa25..86dc1d3c00 100644 --- a/read_buffer/benches/row_group.rs +++ b/read_buffer/benches/row_group.rs @@ -358,13 +358,13 @@ fn generate_trace_for_row_group( let duration_idx = 9; let time_idx = 10; - let env_value = rng.gen_range(0_u8, 2); + let env_value = rng.gen_range(0_u8..2); let env = format!("env-{:?}", env_value); // cardinality of 2. - let data_centre_value = rng.gen_range(0_u8, 10); + let data_centre_value = rng.gen_range(0_u8..10); let data_centre = format!("data_centre-{:?}-{:?}", env_value, data_centre_value); // cardinality of 2 * 10 = 20 - let cluster_value = rng.gen_range(0_u8, 10); + let cluster_value = rng.gen_range(0_u8..10); let cluster = format!( "cluster-{:?}-{:?}-{:?}", env_value, @@ -373,7 +373,7 @@ fn generate_trace_for_row_group( ); // user id is dependent on the cluster - let user_id_value = rng.gen_range(0_u32, 1000); + let user_id_value = rng.gen_range(0_u32..1000); let user_id = format!( "uid-{:?}-{:?}-{:?}-{:?}", env_value, @@ -382,7 +382,7 @@ fn generate_trace_for_row_group( user_id_value // cardinality of 2 * 10 * 10 * 1000 = 200,000 ); - let request_id_value = rng.gen_range(0_u32, 10); + let request_id_value = rng.gen_range(0_u32..10); let request_id = format!( "rid-{:?}-{:?}-{:?}-{:?}-{:?}", env_value, @@ -392,7 +392,11 @@ fn generate_trace_for_row_group( request_id_value // cardinality of 2 * 10 * 10 * 1000 * 10 = 2,000,000 ); - let trace_id = rng.sample_iter(&Alphanumeric).take(8).collect::(); + let trace_id = rng + .sample_iter(&Alphanumeric) + .map(char::from) + .take(8) + .collect::(); // the trace should move across hosts, which in this setup would be nodes // and pods. @@ -401,13 +405,13 @@ fn generate_trace_for_row_group( for _ in 0..spans_per_trace { // these values are not the same for each span so need to be generated // separately. - let node_id = rng.gen_range(0, 10); // cardinality is 2 * 10 * 10 * 10 = 2,000 + let node_id = rng.gen_range(0..10); // cardinality is 2 * 10 * 10 * 10 = 2,000 column_packers[pod_id_idx].str_packer_mut().push(format!( "pod_id-{}-{}-{}", node_id_prefix, node_id, - rng.gen_range(0, 10) // cardinality is 2 * 10 * 10 * 10 * 10 = 20,000 + rng.gen_range(0..10) // cardinality is 2 * 10 * 10 * 10 * 10 = 20,000 )); column_packers[node_id_idx] @@ -415,9 +419,12 @@ fn generate_trace_for_row_group( .push(format!("node_id-{}-{}", node_id_prefix, node_id)); // randomly generate a span_id - column_packers[span_id_idx] - .str_packer_mut() - .push(rng.sample_iter(&Alphanumeric).take(8).collect::()); + column_packers[span_id_idx].str_packer_mut().push( + rng.sample_iter(&Alphanumeric) + .map(char::from) + .take(8) + .collect::(), + ); // randomly generate some duration times in milliseconds. column_packers[duration_idx].i64_packer_mut().push( diff --git a/server/Cargo.toml b/server/Cargo.toml index a68cc4d1fb..8d26d6b735 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,7 +11,7 @@ bytes = "1.0" chrono = "0.4" crc32fast = "1.2.0" data_types = { path = "../data_types" } -flatbuffers = "0.6" +flatbuffers = "0.6" # TODO: Update to 0.8 futures = "0.3.7" generated_types = { path = "../generated_types" } hashbrown = "0.9.1" From 47813176473853cfb2e2150659e11a0d0684de9d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 15 Mar 2021 12:41:18 -0400 Subject: [PATCH 047/104] feat: Management API + CLI command to create a new chunk (rollover partition) (#990) * feat: Management API + CLI command to create a new chunk (rollover partition) * fix: Update tests/end_to_end_cases/management_api.rs fix typo * fix: logical merge conflict Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .../iox/management/v1/service.proto | 16 +++- influxdb_iox_client/src/client/management.rs | 38 ++++++++ mutable_buffer/src/database.rs | 3 +- mutable_buffer/src/partition.rs | 2 + server/src/db.rs | 3 +- src/commands/database/partition.rs | 28 +++++- src/influxdb_ioxd/rpc/management.rs | 23 +++++ tests/end_to_end_cases/management_api.rs | 91 +++++++++++++++++++ tests/end_to_end_cases/management_cli.rs | 58 ++++++++++++ 9 files changed, 258 insertions(+), 4 deletions(-) diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index bb3a4c18cf..5fe8104e4d 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -44,6 +44,9 @@ service ManagementService { // Get detail information about a partition rpc GetPartition(GetPartitionRequest) returns (GetPartitionResponse); + // Create a new chunk in the mutable buffer + rpc NewPartitionChunk(NewPartitionChunkRequest) returns (NewPartitionChunkResponse); + } message GetWriterIdRequest {} @@ -139,7 +142,6 @@ message ListPartitionsResponse { repeated Partition partitions = 1; } - // Request to get details of a specific partition from a named database message GetPartitionRequest { // the name of the database @@ -153,3 +155,15 @@ message GetPartitionResponse { // Detailed information about a partition Partition partition = 1; } + +// Request that a new chunk for writing is created in the mutable buffer +message NewPartitionChunkRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; +} + +message NewPartitionChunkResponse { +} diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index 8b559bde45..f47369b924 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -132,8 +132,23 @@ pub enum GetPartitionError { ServerError(tonic::Status), } +/// Errors returned by Client::new_partition_chunk +#[derive(Debug, Error)] +pub enum NewPartitionChunkError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// An IOx Management API client. /// +/// This client wraps the underlying `tonic` generated client with a +/// more ergonomic interface. +/// /// ```no_run /// #[tokio::main] /// # async fn main() { @@ -348,4 +363,27 @@ impl Client { partition.ok_or(GetPartitionError::PartitionNotFound) } + + /// Create a new chunk in a partittion + pub async fn new_partition_chunk( + &mut self, + db_name: impl Into, + partition_key: impl Into, + ) -> Result<(), NewPartitionChunkError> { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + self.inner + .new_partition_chunk(NewPartitionChunkRequest { + db_name, + partition_key, + }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => NewPartitionChunkError::DatabaseNotFound, + _ => NewPartitionChunkError::ServerError(status), + })?; + + Ok(()) + } } diff --git a/mutable_buffer/src/database.rs b/mutable_buffer/src/database.rs index 200b64c307..1e095f52ff 100644 --- a/mutable_buffer/src/database.rs +++ b/mutable_buffer/src/database.rs @@ -104,7 +104,8 @@ impl MutableBufferDb { Ok(()) } - /// Rolls over the active chunk in this partititon + /// Rolls over the active chunk in this partititon. Returns the + /// previously open (now closed) Chunk pub fn rollover_partition(&self, partition_key: &str) -> Result> { let partition = self.get_partition(partition_key); let mut partition = partition.write().expect("mutex poisoned"); diff --git a/mutable_buffer/src/partition.rs b/mutable_buffer/src/partition.rs index 0d588cadd0..f050199cae 100644 --- a/mutable_buffer/src/partition.rs +++ b/mutable_buffer/src/partition.rs @@ -178,6 +178,8 @@ impl Partition { /// /// Queries will continue to see data in the specified chunk until /// it is dropped. + /// + /// Returns the previously open (now closed) Chunk pub fn rollover_chunk(&mut self) -> Arc { let chunk_id = self.id_generator; self.id_generator += 1; diff --git a/server/src/db.rs b/server/src/db.rs index e026994162..fc05d83c82 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -117,7 +117,8 @@ impl Db { } } - /// Rolls over the active chunk in the database's specified partition + /// Rolls over the active chunk in the database's specified + /// partition. Returns the previously open (now closed) Chunk pub async fn rollover_partition(&self, partition_key: &str) -> Result> { if let Some(local_store) = self.mutable_buffer.as_ref() { local_store diff --git a/src/commands/database/partition.rs b/src/commands/database/partition.rs index 7aed0698ab..39c4df3f4a 100644 --- a/src/commands/database/partition.rs +++ b/src/commands/database/partition.rs @@ -1,7 +1,7 @@ //! This module implements the `partition` CLI command use influxdb_iox_client::{ connection::Builder, - management::{self, GetPartitionError, ListPartitionsError}, + management::{self, GetPartitionError, ListPartitionsError, NewPartitionChunkError}, }; use structopt::StructOpt; use thiserror::Error; @@ -14,6 +14,9 @@ pub enum Error { #[error("Error getting partition: {0}")] GetPartitionsError(#[from] GetPartitionError), + #[error("Error getting partition: {0}")] + NewPartitionError(#[from] NewPartitionChunkError), + #[error("Error rendering response as JSON: {0}")] WritingJson(#[from] serde_json::Error), @@ -49,6 +52,17 @@ struct Get { partition_key: String, } +/// Create a new, open chunk in the partiton's Mutable Buffer which will receive +/// new writes. +#[derive(Debug, StructOpt)] +struct NewChunk { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, +} + /// All possible subcommands for partition #[derive(Debug, StructOpt)] enum Command { @@ -56,6 +70,8 @@ enum Command { List(List), // Get details about a particular partition Get(Get), + // Create a new chunk in the partition + NewChunk(NewChunk), } pub async fn command(url: String, config: Config) -> Result<()> { @@ -91,6 +107,16 @@ pub async fn command(url: String, config: Config) -> Result<()> { serde_json::to_writer_pretty(std::io::stdout(), &partition_detail)?; } + Command::NewChunk(new_chunk) => { + let NewChunk { + db_name, + partition_key, + } = new_chunk; + + // Ignore response for now + client.new_partition_chunk(db_name, partition_key).await?; + println!("Ok"); + } } Ok(()) diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index 0e089d97de..d54b26b327 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -233,6 +233,29 @@ where Ok(Response::new(GetPartitionResponse { partition })) } + + async fn new_partition_chunk( + &self, + request: Request, + ) -> Result, Status> { + let NewPartitionChunkRequest { + db_name, + partition_key, + } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + db.rollover_partition(&partition_key) + .await + .map_err(default_db_error_handler)?; + + Ok(Response::new(NewPartitionChunkResponse {})) + } } pub fn make_server( diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 7be531b09c..b9079a76ad 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -391,3 +391,94 @@ async fn test_partition_get_error() { assert_contains!(err.to_string(), "Partition not found"); } + +#[tokio::test] +async fn test_new_partition_chunk() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = vec!["cpu,region=west user=23.2 100"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + assert_eq!(chunks.len(), 1, "Chunks: {:#?}", chunks); + let partition_key = "cpu"; + + // Rollover the a second chunk + management_client + .new_partition_chunk(&db_name, partition_key) + .await + .expect("new partition chunk"); + + // Load some more data and now expect that we have a second chunk + + let lp_lines = vec!["cpu,region=west user=21.0 150"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + assert_eq!(chunks.len(), 2, "Chunks: {:#?}", chunks); + + // Made all chunks in the same partition + assert_eq!( + chunks.iter().filter(|c| c.partition_key == "cpu").count(), + 2, + "Chunks: {:#?}", + chunks + ); + + // Rollover a (currently non existent) partition which is OK + management_client + .new_partition_chunk(&db_name, "non_existent_partition") + .await + .expect("new partition chunk"); + + assert_eq!(chunks.len(), 2, "Chunks: {:#?}", chunks); + assert_eq!( + chunks.iter().filter(|c| c.partition_key == "cpu").count(), + 2, + "Chunks: {:#?}", + chunks + ); + assert_eq!( + chunks + .iter() + .filter(|c| c.partition_key == "non_existent_partition") + .count(), + 0, + "Chunks: {:#?}", + chunks + ); +} + +#[tokio::test] +async fn test_new_partition_chunk_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + + let err = management_client + .new_partition_chunk("this database does not exist", "nor_does_this_partition") + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); +} diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 97c0357d61..a690f6ced3 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -377,6 +377,64 @@ async fn test_get_partition_error() { .stderr(predicate::str::contains("Database not found")); } +#[tokio::test] +async fn test_new_partition_chunk() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec!["cpu,region=west user=23.2 100"]; + load_lp(addr, &db_name, lp_data); + + let expected = "Ok"; + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("new-chunk") + .arg(&db_name) + .arg("cpu") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); + + let expected = "ClosedMutableBuffer"; + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("chunk") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +#[tokio::test] +async fn test_new_partition_chunk_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("new-chunk") + .arg("non_existent_database") + .arg("non_existent_partition") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + /// Loads the specified lines into the named database fn load_lp(addr: &str, db_name: &str, lp_data: Vec<&str>) { let lp_data_file = make_temp_file(lp_data.join("\n")); From 58b01b4ad3614df507ef3d16fa5363c211a581d2 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 15 Mar 2021 17:15:05 +0100 Subject: [PATCH 048/104] fix: Add ca-certificates to iox docker image --- Dockerfile | 2 +- docker/Dockerfile.iox | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 46b9d2a192..bcbf98dd29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN \ FROM debian:buster-slim RUN apt-get update \ - && apt-get install -y libssl1.1 libgcc1 libc6 --no-install-recommends \ + && apt-get install -y libssl1.1 libgcc1 libc6 ca-certificates --no-install-recommends \ && rm -rf /var/lib/{apt,dpkg,cache,log} RUN groupadd -g 1500 rust \ diff --git a/docker/Dockerfile.iox b/docker/Dockerfile.iox index ffa68a7092..2e13c4ebf8 100644 --- a/docker/Dockerfile.iox +++ b/docker/Dockerfile.iox @@ -4,7 +4,7 @@ FROM debian:buster-slim RUN apt-get update \ - && apt-get install -y libssl1.1 libgcc1 libc6 \ + && apt-get install -y libssl1.1 libgcc1 libc6 ca-certificates --no-install-recommends \ && rm -rf /var/lib/{apt,dpkg,cache,log} RUN groupadd -g 1500 rust \ From a3c146a38e2669f50b2d7bbb2f9602f7bba0c823 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 16 Mar 2021 08:40:40 -0400 Subject: [PATCH 049/104] docs: Update README.md to refer to the command and CLI help (#982) * docs: Update README.md to refer to the command and CLI help * docs: Apply suggestions from code review Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- README.md | 9 +++++++++ src/main.rs | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/README.md b/README.md index 1468a42a4c..75ea410db0 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,15 @@ To query data stored in the `company_sensors` database: influxdb_iox database query company_sensors "SELECT * FROM cpu LIMIT 10" ``` +### Using the CLI + +To ease deloyment, IOx is packaged as a combined binary which has +commands to start the IOx server as well as a CLI interface for +interacting with and configuring such servers. + +The CLI itself is documented via extensive built in help which you can +access by runing `influxdb_iox --help` + ### InfluxDB 2.0 compatibility diff --git a/src/main.rs b/src/main.rs index 48ade670df..6429424ff9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,6 +63,14 @@ Examples: # Dumps storage statistics about out.parquet to stdout influxdb_iox stats out.parquet + +Command are generally structured in the form: + + +For example, a command such as the following shows all actions + available for database chunks, including get and list. + + influxdb_iox database chunk --help "# )] struct Config { From 51304fd138f3bbfdb53b94972d0bfd979c955b30 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 16 Mar 2021 08:53:01 -0400 Subject: [PATCH 050/104] docs: Add README for influxdb_client (#996) * docs: Add README for influxdb_client * docs: Update influxdb2_client/README.md Co-authored-by: Paul Dix * docs: Apply suggestions from code review Co-authored-by: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Co-authored-by: Paul Dix Co-authored-by: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- influxdb2_client/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 influxdb2_client/README.md diff --git a/influxdb2_client/README.md b/influxdb2_client/README.md new file mode 100644 index 0000000000..ce4842a8ea --- /dev/null +++ b/influxdb2_client/README.md @@ -0,0 +1,23 @@ +# InfluxDB V2 Client API + +This crate contains a work-in-progress implementation of a Rust client for the [InfluxDB 2.0 API](https://docs.influxdata.com/influxdb/v2.0/reference/api/). + +This client is not the Rust client for IOx. You can find that [here](../influxdb_iox_client). + +The InfluxDB IOx project plans to focus its efforts on the subset of the API which are most relevent to IOx, but we accept (welcome!) PRs for adding the other pieces of functionality. + + +## Design Notes + +When it makes sense, this client aims to mirror the [InfluxDB 2.x Go client API](https://github.com/influxdata/influxdb-client-go) + +## Contributing + +If you would like to contribute code you can do through GitHub by forking the repository and sending a pull request into the master branch. + + +## Future work + +- [ ] Publish as a crate on [crates.io](http://crates.io) + +If you would like to contribute code you can do through GitHub by forking the repository and sending a pull request into the main branch. From 9c112b98723c279a11bafe66ba3fafde80d7089f Mon Sep 17 00:00:00 2001 From: Aakash Hemadri Date: Tue, 16 Mar 2021 18:36:20 +0530 Subject: [PATCH 051/104] feat: Add support for ready API to influxdb_client (#999) Get the readiness of an instance at startup Signed-off-by: Aakash Hemadri Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- influxdb2_client/examples/ready.rs | 11 +++++++ influxdb2_client/src/lib.rs | 2 ++ influxdb2_client/src/ready.rs | 51 ++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 influxdb2_client/examples/ready.rs create mode 100644 influxdb2_client/src/ready.rs diff --git a/influxdb2_client/examples/ready.rs b/influxdb2_client/examples/ready.rs new file mode 100644 index 0000000000..07d69c4d88 --- /dev/null +++ b/influxdb2_client/examples/ready.rs @@ -0,0 +1,11 @@ +#[tokio::main] +async fn main() -> Result<(), Box> { + let influx_url = "some-url"; + let token = "some-token"; + + let client = influxdb2_client::Client::new(influx_url, token); + + println!("{:?}", client.ready().await?); + + Ok(()) +} diff --git a/influxdb2_client/src/lib.rs b/influxdb2_client/src/lib.rs index 52a1b60e6a..f332a0a154 100644 --- a/influxdb2_client/src/lib.rs +++ b/influxdb2_client/src/lib.rs @@ -302,3 +302,5 @@ cpu,host=server01,region=us-west usage=0.87 Ok(()) } } + +mod ready; diff --git a/influxdb2_client/src/ready.rs b/influxdb2_client/src/ready.rs new file mode 100644 index 0000000000..fae9ecce94 --- /dev/null +++ b/influxdb2_client/src/ready.rs @@ -0,0 +1,51 @@ +use reqwest::{Method, StatusCode}; +use snafu::ResultExt; + +use super::{Client, Http, RequestError, ReqwestProcessing}; + +impl Client { + /// Get the readiness of an instance at startup + pub async fn ready(&self) -> Result { + let ready_url = format!("{}/ready", self.url); + let response = self + .request(Method::GET, &ready_url) + .send() + .await + .context(ReqwestProcessing)?; + + match response.status() { + StatusCode::OK => Ok(true), + _ => { + let status = response.status(); + let text = response.text().await.context(ReqwestProcessing)?; + Http { status, text }.fail()? + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mockito::mock; + + type Error = Box; + type Result = std::result::Result; + + #[tokio::test] + async fn ready() -> Result { + let token = "some-token"; + + let mock_server = mock("GET", "/ready") + .match_header("Authorization", format!("Token {}", token).as_str()) + .create(); + + let client = Client::new(&mockito::server_url(), token); + + let _result = client.ready().await; + + mock_server.assert(); + + Ok(()) + } +} From 3fe1b8c5b73338c225ef9fba827f4df807612dd8 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Tue, 16 Mar 2021 13:19:44 +0000 Subject: [PATCH 052/104] feat: add longrunning operations client (#981) refactor: add separate format feature influxdb_iox_client Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- Cargo.lock | 1 + Cargo.toml | 2 +- data_types/Cargo.toml | 1 + data_types/src/job.rs | 75 ++++++++++- generated_types/src/lib.rs | 58 +++++++++ influxdb_iox_client/Cargo.toml | 1 + influxdb_iox_client/src/client.rs | 3 + influxdb_iox_client/src/client/management.rs | 32 +++++ influxdb_iox_client/src/client/operations.rs | 125 +++++++++++++++++++ influxdb_iox_client/src/lib.rs | 6 +- src/commands/operations.rs | 113 +++++++++++++++++ src/influxdb_ioxd/rpc/operations.rs | 3 +- src/main.rs | 9 ++ tests/common/server_fixture.rs | 6 + tests/end_to_end_cases/mod.rs | 2 + tests/end_to_end_cases/operations_api.rs | 94 ++++++++++++++ tests/end_to_end_cases/operations_cli.rs | 66 ++++++++++ 17 files changed, 590 insertions(+), 7 deletions(-) create mode 100644 influxdb_iox_client/src/client/operations.rs create mode 100644 src/commands/operations.rs create mode 100644 tests/end_to_end_cases/operations_api.rs create mode 100644 tests/end_to_end_cases/operations_cli.rs diff --git a/Cargo.lock b/Cargo.lock index 9b1893c62e..a3d63943c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -755,6 +755,7 @@ dependencies = [ "generated_types", "influxdb_line_protocol", "percent-encoding", + "prost", "regex", "serde", "serde_regex", diff --git a/Cargo.toml b/Cargo.toml index 8d339c813d..c576c002b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ debug = true arrow_deps = { path = "arrow_deps" } data_types = { path = "data_types" } generated_types = { path = "generated_types" } -influxdb_iox_client = { path = "influxdb_iox_client" } +influxdb_iox_client = { path = "influxdb_iox_client", features = ["format"] } influxdb_line_protocol = { path = "influxdb_line_protocol" } influxdb_tsm = { path = "influxdb_tsm" } ingest = { path = "ingest" } diff --git a/data_types/Cargo.toml b/data_types/Cargo.toml index b2c669ae26..96dcb83659 100644 --- a/data_types/Cargo.toml +++ b/data_types/Cargo.toml @@ -12,6 +12,7 @@ flatbuffers = "0.6" # TODO: Update to 0.8 generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } percent-encoding = "2.1.0" +prost = "0.7" serde = "1.0" snafu = "0.6" tracing = "0.1" diff --git a/data_types/src/job.rs b/data_types/src/job.rs index 738b39a938..9540fb6762 100644 --- a/data_types/src/job.rs +++ b/data_types/src/job.rs @@ -1,8 +1,15 @@ -use generated_types::influxdata::iox::management::v1 as management; +use generated_types::google::{protobuf::Any, FieldViolation, FieldViolationExt}; +use generated_types::{ + google::longrunning, influxdata::iox::management::v1 as management, protobuf_type_url_eq, +}; +use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; /// Metadata associated with a set of background tasks /// Used in combination with TrackerRegistry -#[derive(Debug, Clone)] +/// +/// TODO: Serde is temporary until prost adds JSON support +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum Job { PersistSegment { writer_id: u32, segment_id: u64 }, Dummy { nanos: Vec }, @@ -22,3 +29,67 @@ impl From for management::operation_metadata::Job { } } } + +impl From for Job { + fn from(value: management::operation_metadata::Job) -> Self { + use management::operation_metadata::Job; + match value { + Job::Dummy(management::Dummy { nanos }) => Self::Dummy { nanos }, + Job::PersistSegment(management::PersistSegment { + writer_id, + segment_id, + }) => Self::PersistSegment { + writer_id, + segment_id, + }, + } + } +} + +/// A group of asynchronous tasks being performed by an IOx server +/// +/// TODO: Temporary until prost adds JSON support +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Operation { + /// ID of the running operation + pub id: usize, + /// Number of subtasks for this operation + pub task_count: u64, + /// Number of pending tasks for this operation + pub pending_count: u64, + /// Wall time spent executing this operation + pub wall_time: std::time::Duration, + /// CPU time spent executing this operation + pub cpu_time: std::time::Duration, + /// Additional job metadata + pub job: Option, +} + +impl TryFrom for Operation { + type Error = FieldViolation; + + fn try_from(operation: longrunning::Operation) -> Result { + let metadata: Any = operation + .metadata + .ok_or_else(|| FieldViolation::required("metadata"))?; + + if !protobuf_type_url_eq(&metadata.type_url, management::OPERATION_METADATA) { + return Err(FieldViolation { + field: "metadata.type_url".to_string(), + description: "Unexpected field type".to_string(), + }); + } + + let meta: management::OperationMetadata = + prost::Message::decode(metadata.value).field("metadata.value")?; + + Ok(Self { + id: operation.name.parse().field("name")?, + task_count: meta.task_count, + pending_count: meta.pending_count, + wall_time: std::time::Duration::from_nanos(meta.wall_nanos), + cpu_time: std::time::Duration::from_nanos(meta.cpu_nanos), + job: meta.job.map(Into::into), + }) + } +} diff --git a/generated_types/src/lib.rs b/generated_types/src/lib.rs index f06c7feaff..b97ba87bcd 100644 --- a/generated_types/src/lib.rs +++ b/generated_types/src/lib.rs @@ -32,6 +32,10 @@ pub mod influxdata { pub mod iox { pub mod management { pub mod v1 { + /// Operation metadata type + pub const OPERATION_METADATA: &str = + "influxdata.iox.management.v1.OperationMetadata"; + include!(concat!(env!("OUT_DIR"), "/influxdata.iox.management.v1.rs")); } } @@ -78,8 +82,62 @@ pub const STORAGE_SERVICE: &str = "influxdata.platform.storage.Storage"; pub const IOX_TESTING_SERVICE: &str = "influxdata.platform.storage.IOxTesting"; /// gRPC Arrow Flight Service pub const ARROW_SERVICE: &str = "arrow.flight.protocol.FlightService"; +/// The type prefix for any types +pub const ANY_TYPE_PREFIX: &str = "type.googleapis.com"; + +/// Returns the protobuf URL usable with a google.protobuf.Any message +/// This is the full Protobuf package and message name prefixed by +/// "type.googleapis.com/" +pub fn protobuf_type_url(protobuf_type: &str) -> String { + format!("{}/{}", ANY_TYPE_PREFIX, protobuf_type) +} + +/// Compares the protobuf type URL found within a google.protobuf.Any +/// message to an expected Protobuf package and message name +/// +/// i.e. strips off the "type.googleapis.com/" prefix from `url` +/// and compares the result with `protobuf_type` +/// +/// ``` +/// use generated_types::protobuf_type_url_eq; +/// assert!(protobuf_type_url_eq("type.googleapis.com/google.protobuf.Empty", "google.protobuf.Empty")); +/// assert!(!protobuf_type_url_eq("type.googleapis.com/google.protobuf.Empty", "something.else")); +/// ``` +pub fn protobuf_type_url_eq(url: &str, protobuf_type: &str) -> bool { + let mut split = url.splitn(2, '/'); + match (split.next(), split.next()) { + (Some(ANY_TYPE_PREFIX), Some(t)) => t == protobuf_type, + _ => false, + } +} pub use com::github::influxdata::idpe::storage::read::*; pub use influxdata::platform::storage::*; pub mod google; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_protobuf_type_url() { + use influxdata::iox::management::v1::OPERATION_METADATA; + + let t = protobuf_type_url(OPERATION_METADATA); + + assert_eq!( + &t, + "type.googleapis.com/influxdata.iox.management.v1.OperationMetadata" + ); + + assert!(protobuf_type_url_eq(&t, OPERATION_METADATA)); + assert!(!protobuf_type_url_eq(&t, "foo")); + + // The URL must start with the type.googleapis.com prefix + assert!(!protobuf_type_url_eq( + OPERATION_METADATA, + OPERATION_METADATA + )); + } +} diff --git a/influxdb_iox_client/Cargo.toml b/influxdb_iox_client/Cargo.toml index 1dc3fb8d34..12f2383f44 100644 --- a/influxdb_iox_client/Cargo.toml +++ b/influxdb_iox_client/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [features] flight = ["arrow_deps", "serde/derive", "serde_json", "futures-util"] +format = ["arrow_deps"] [dependencies] # Workspace dependencies, in alphabetical order diff --git a/influxdb_iox_client/src/client.rs b/influxdb_iox_client/src/client.rs index 53ba6ead6c..cf822b523e 100644 --- a/influxdb_iox_client/src/client.rs +++ b/influxdb_iox_client/src/client.rs @@ -7,6 +7,9 @@ pub mod management; /// Client for the write API pub mod write; +/// Client for the operations API +pub mod operations; + #[cfg(feature = "flight")] /// Client for the flight API pub mod flight; diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index f47369b924..759ae53502 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -5,6 +5,7 @@ use thiserror::Error; use self::generated_types::{management_service_client::ManagementServiceClient, *}; use crate::connection::Connection; +use ::generated_types::google::longrunning::Operation; use std::convert::TryInto; /// Re-export generated_types @@ -104,6 +105,18 @@ pub enum UpdateRemoteError { ServerError(tonic::Status), } +/// Errors returned by Client::create_dummy_job +#[derive(Debug, Error)] +pub enum CreateDummyJobError { + /// Response contained no payload + #[error("Server returned an empty response")] + EmptyResponse, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// Errors returned by Client::list_partitions #[derive(Debug, Error)] pub enum ListPartitionsError { @@ -386,4 +399,23 @@ impl Client { Ok(()) } + + /// Creates a dummy job that for each value of the nanos field + /// spawns a task that sleeps for that number of nanoseconds before + /// returning + pub async fn create_dummy_job( + &mut self, + nanos: Vec, + ) -> Result { + let response = self + .inner + .create_dummy_job(CreateDummyJobRequest { nanos }) + .await + .map_err(CreateDummyJobError::ServerError)?; + + Ok(response + .into_inner() + .operation + .ok_or(CreateDummyJobError::EmptyResponse)?) + } } diff --git a/influxdb_iox_client/src/client/operations.rs b/influxdb_iox_client/src/client/operations.rs new file mode 100644 index 0000000000..462baeb0b3 --- /dev/null +++ b/influxdb_iox_client/src/client/operations.rs @@ -0,0 +1,125 @@ +use thiserror::Error; + +use ::generated_types::google::FieldViolation; + +use crate::connection::Connection; + +use self::generated_types::{operations_client::OperationsClient, *}; + +/// Re-export generated_types +pub mod generated_types { + pub use generated_types::google::longrunning::*; +} + +/// Error type for the operations Client +#[derive(Debug, Error)] +pub enum Error { + /// Client received an invalid response + #[error("Invalid server response: {}", .0)] + InvalidResponse(#[from] FieldViolation), + + /// Operation was not found + #[error("Operation not found: {}", .0)] + NotFound(usize), + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Result type for the operations Client +pub type Result = std::result::Result; + +/// An IOx Long Running Operations API client. +/// +/// ```no_run +/// #[tokio::main] +/// # async fn main() { +/// use influxdb_iox_client::{ +/// operations::Client, +/// connection::Builder, +/// }; +/// +/// let mut connection = Builder::default() +/// .build("http://127.0.0.1:8082") +/// .await +/// .unwrap(); +/// +/// let mut client = Client::new(connection); +/// # } +/// ``` +#[derive(Debug, Clone)] +pub struct Client { + inner: OperationsClient, +} + +impl Client { + /// Creates a new client with the provided connection + pub fn new(channel: tonic::transport::Channel) -> Self { + Self { + inner: OperationsClient::new(channel), + } + } + + /// Get information about all operations + pub async fn list_operations(&mut self) -> Result> { + Ok(self + .inner + .list_operations(ListOperationsRequest::default()) + .await + .map_err(Error::ServerError)? + .into_inner() + .operations) + } + + /// Get information about a specific operation + pub async fn get_operation(&mut self, id: usize) -> Result { + Ok(self + .inner + .get_operation(GetOperationRequest { + name: id.to_string(), + }) + .await + .map_err(|e| match e.code() { + tonic::Code::NotFound => Error::NotFound(id), + _ => Error::ServerError(e), + })? + .into_inner()) + } + + /// Cancel a given operation + pub async fn cancel_operation(&mut self, id: usize) -> Result<()> { + self.inner + .cancel_operation(CancelOperationRequest { + name: id.to_string(), + }) + .await + .map_err(|e| match e.code() { + tonic::Code::NotFound => Error::NotFound(id), + _ => Error::ServerError(e), + })?; + + Ok(()) + } + + /// Waits until an operation completes, or the timeout expires, and + /// returns the latest operation metadata + pub async fn wait_operation( + &mut self, + id: usize, + timeout: Option, + ) -> Result { + Ok(self + .inner + .wait_operation(WaitOperationRequest { + name: id.to_string(), + timeout: timeout.map(Into::into), + }) + .await + .map_err(|e| match e.code() { + tonic::Code::NotFound => Error::NotFound(id), + _ => Error::ServerError(e), + })? + .into_inner()) + } +} diff --git a/influxdb_iox_client/src/lib.rs b/influxdb_iox_client/src/lib.rs index 855e41a39c..1f2ab0dc45 100644 --- a/influxdb_iox_client/src/lib.rs +++ b/influxdb_iox_client/src/lib.rs @@ -8,14 +8,14 @@ )] #![allow(clippy::missing_docs_in_private_items)] -pub use client::{health, management, write}; +pub use generated_types::{protobuf_type_url, protobuf_type_url_eq}; -#[cfg(feature = "flight")] -pub use client::flight; +pub use client::*; /// Builder for constructing connections for use with the various gRPC clients pub mod connection; +#[cfg(feature = "format")] /// Output formatting utilities pub mod format; diff --git a/src/commands/operations.rs b/src/commands/operations.rs new file mode 100644 index 0000000000..037353c5dd --- /dev/null +++ b/src/commands/operations.rs @@ -0,0 +1,113 @@ +use data_types::job::Operation; +use generated_types::google::FieldViolation; +use influxdb_iox_client::{ + connection::Builder, + management, + operations::{self, Client}, +}; +use std::convert::TryInto; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), + + #[error("Client error: {0}")] + ClientError(#[from] operations::Error), + + #[error("Received invalid response: {0}")] + InvalidResponse(#[from] FieldViolation), + + #[error("Failed to create dummy job: {0}")] + CreateDummyJobError(#[from] management::CreateDummyJobError), + + #[error("Output serialization error: {0}")] + SerializationError(#[from] serde_json::Error), +} + +pub type Result = std::result::Result; + +/// Manage long-running IOx operations +#[derive(Debug, StructOpt)] +pub struct Config { + #[structopt(subcommand)] + command: Command, +} + +#[derive(Debug, StructOpt)] +enum Command { + /// Get list of running operations + List, + + /// Get a specific operation + Get { + /// The id of the operation + id: usize, + }, + + /// Wait for a specific operation to complete + Wait { + /// The id of the operation + id: usize, + + /// Maximum number of nanoseconds to wait before returning current + /// status + nanos: Option, + }, + + /// Cancel a specific operation + Cancel { + /// The id of the operation + id: usize, + }, + + /// Spawns a dummy test operation + Test { nanos: Vec }, +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + + match config.command { + Command::List => { + let result: Result, _> = Client::new(connection) + .list_operations() + .await? + .into_iter() + .map(TryInto::try_into) + .collect(); + let operations = result?; + serde_json::to_writer_pretty(std::io::stdout(), &operations)?; + } + Command::Get { id } => { + let operation: Operation = Client::new(connection) + .get_operation(id) + .await? + .try_into()?; + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } + Command::Wait { id, nanos } => { + let timeout = nanos.map(std::time::Duration::from_nanos); + let operation: Operation = Client::new(connection) + .wait_operation(id, timeout) + .await? + .try_into()?; + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } + Command::Cancel { id } => { + Client::new(connection).cancel_operation(id).await?; + println!("Ok"); + } + Command::Test { nanos } => { + let operation: Operation = management::Client::new(connection) + .create_dummy_job(nanos) + .await? + .try_into()?; + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } + } + + Ok(()) +} diff --git a/src/influxdb_ioxd/rpc/operations.rs b/src/influxdb_ioxd/rpc/operations.rs index 9bacc595ae..871e020ff2 100644 --- a/src/influxdb_ioxd/rpc/operations.rs +++ b/src/influxdb_ioxd/rpc/operations.rs @@ -16,6 +16,7 @@ use generated_types::{ FieldViolation, InternalError, NotFound, }, influxdata::iox::management::v1 as management, + protobuf_type_url, }; use server::{ tracker::{Tracker, TrackerId}, @@ -48,7 +49,7 @@ pub fn encode_tracker(tracker: Tracker) -> Result })?; let metadata = Any { - type_url: "type.googleapis.com/influxdata.iox.management.v1.OperationMetadata".to_string(), + type_url: protobuf_type_url(management::OPERATION_METADATA), value: buffer.freeze(), }; diff --git a/src/main.rs b/src/main.rs index 6429424ff9..cd2146b27a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,7 @@ mod commands { mod input; pub mod logging; pub mod meta; + pub mod operations; pub mod run; pub mod server; pub mod server_remote; @@ -124,6 +125,7 @@ enum Command { Stats(commands::stats::Config), Server(commands::server::Config), Writer(commands::writer::Config), + Operation(commands::operations::Config), } fn main() -> Result<(), std::io::Error> { @@ -193,6 +195,13 @@ fn main() -> Result<(), std::io::Error> { std::process::exit(ReturnCode::Failure as _) } } + Command::Operation(config) => { + logging_level.setup_basic_logging(); + if let Err(e) = commands::operations::command(host, config).await { + eprintln!("{}", e); + std::process::exit(ReturnCode::Failure as _) + } + } Command::Server(config) => { logging_level.setup_basic_logging(); if let Err(e) = commands::server::command(host, config).await { diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs index 83f157ed7c..2f3c07257d 100644 --- a/tests/common/server_fixture.rs +++ b/tests/common/server_fixture.rs @@ -181,6 +181,12 @@ impl ServerFixture { influxdb_iox_client::management::Client::new(self.grpc_channel()) } + /// Return a operations client suitable for communicating with this + /// server + pub fn operations_client(&self) -> influxdb_iox_client::operations::Client { + influxdb_iox_client::operations::Client::new(self.grpc_channel()) + } + /// Return a write client suitable for communicating with this /// server pub fn write_client(&self) -> influxdb_iox_client::write::Client { diff --git a/tests/end_to_end_cases/mod.rs b/tests/end_to_end_cases/mod.rs index e21f0dde4f..776420167a 100644 --- a/tests/end_to_end_cases/mod.rs +++ b/tests/end_to_end_cases/mod.rs @@ -2,6 +2,8 @@ pub mod flight_api; pub mod http; pub mod management_api; pub mod management_cli; +pub mod operations_api; +pub mod operations_cli; pub mod read_api; pub mod read_cli; pub mod scenario; diff --git a/tests/end_to_end_cases/operations_api.rs b/tests/end_to_end_cases/operations_api.rs new file mode 100644 index 0000000000..d1024c9ed6 --- /dev/null +++ b/tests/end_to_end_cases/operations_api.rs @@ -0,0 +1,94 @@ +use crate::common::server_fixture::ServerFixture; +use generated_types::google::protobuf::Any; +use influxdb_iox_client::{management::generated_types::*, operations, protobuf_type_url_eq}; +use std::time::Duration; + +fn get_operation_metadata(metadata: Option) -> OperationMetadata { + assert!(metadata.is_some()); + let metadata = metadata.unwrap(); + assert!(protobuf_type_url_eq(&metadata.type_url, OPERATION_METADATA)); + prost::Message::decode(metadata.value).expect("failed to decode metadata") +} + +#[tokio::test] +async fn test_operations() { + let server_fixture = ServerFixture::create_single_use().await; + let mut management_client = server_fixture.management_client(); + let mut operations_client = server_fixture.operations_client(); + + let running_ops = operations_client + .list_operations() + .await + .expect("list operations failed"); + + assert_eq!(running_ops.len(), 0); + + let nanos = vec![Duration::from_secs(20).as_nanos() as _, 1]; + + let operation = management_client + .create_dummy_job(nanos.clone()) + .await + .expect("create dummy job failed"); + + let running_ops = operations_client + .list_operations() + .await + .expect("list operations failed"); + + assert_eq!(running_ops.len(), 1); + assert_eq!(running_ops[0].name, operation.name); + + let id = operation.name.parse().expect("not an integer"); + + // Check we can fetch metadata for Operation + let fetched = operations_client + .get_operation(id) + .await + .expect("get operation failed"); + let meta = get_operation_metadata(fetched.metadata); + let job = meta.job.expect("expected a job"); + + assert_eq!(meta.task_count, 2); + assert_eq!(meta.pending_count, 1); + assert_eq!(job, operation_metadata::Job::Dummy(Dummy { nanos })); + assert!(!fetched.done); + + // Check wait times out correctly + let fetched = operations_client + .wait_operation(id, Some(Duration::from_micros(10))) + .await + .expect("failed to wait operation"); + + assert!(!fetched.done); + // Shouldn't specify wall_nanos as not complete + assert_eq!(meta.wall_nanos, 0); + + let wait = tokio::spawn(async move { + let mut operations_client = server_fixture.operations_client(); + operations_client + .wait_operation(id, None) + .await + .expect("failed to wait operation") + }); + + operations_client + .cancel_operation(id) + .await + .expect("failed to cancel operation"); + + let waited = wait.await.unwrap(); + let meta = get_operation_metadata(waited.metadata); + + assert!(waited.done); + assert!(meta.wall_nanos > 0); + assert!(meta.cpu_nanos > 0); + assert_eq!(meta.pending_count, 0); + assert_eq!(meta.task_count, 2); + + match waited.result { + Some(operations::generated_types::operation::Result::Error(status)) => { + assert_eq!(status.code, tonic::Code::Cancelled as i32) + } + _ => panic!("expected error"), + } +} diff --git a/tests/end_to_end_cases/operations_cli.rs b/tests/end_to_end_cases/operations_cli.rs new file mode 100644 index 0000000000..57495d7ab9 --- /dev/null +++ b/tests/end_to_end_cases/operations_cli.rs @@ -0,0 +1,66 @@ +use crate::common::server_fixture::ServerFixture; +use assert_cmd::Command; +use data_types::job::{Job, Operation}; +use predicates::prelude::*; + +#[tokio::test] +async fn test_start_stop() { + let server_fixture = ServerFixture::create_single_use().await; + let addr = server_fixture.grpc_base(); + let duration = std::time::Duration::from_secs(10).as_nanos() as u64; + + let stdout: Operation = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("test") + .arg(duration.to_string()) + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("expected JSON output"); + + assert_eq!(stdout.task_count, 1); + match stdout.job { + Some(Job::Dummy { nanos }) => assert_eq!(nanos, vec![duration]), + _ => panic!("expected dummy job got {:?}", stdout.job), + } + + let stdout: Vec = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("expected JSON output"); + + assert_eq!(stdout.len(), 1); + match &stdout[0].job { + Some(Job::Dummy { nanos }) => { + assert_eq!(nanos.len(), 1); + assert_eq!(nanos[0], duration); + } + _ => panic!("expected dummy job got {:?}", &stdout[0].job), + } + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("cancel") + .arg(stdout[0].id.to_string()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); +} From ef9c3f3d8faba2ba6f52dfd3c334cf7da206e99e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 16 Mar 2021 16:10:55 -0400 Subject: [PATCH 053/104] feat: Management API + CLI command to list chunks in a partition (#995) * feat: Management API + CLI command to list chunks in a partition For ease of use * refactor: remove unecessary Result --- .../iox/management/v1/service.proto | 21 +++++- influxdb_iox_client/src/client/management.rs | 28 ++++++++ server/src/db.rs | 59 +++++++++++++++-- src/commands/database/partition.rs | 45 ++++++++++++- src/influxdb_ioxd/rpc/management.rs | 24 +++++++ tests/end_to_end_cases/management_api.rs | 57 +++++++++++++++++ tests/end_to_end_cases/management_cli.rs | 64 +++++++++++++++++++ 7 files changed, 287 insertions(+), 11 deletions(-) diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index 5fe8104e4d..d53b225fc7 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -44,6 +44,9 @@ service ManagementService { // Get detail information about a partition rpc GetPartition(GetPartitionRequest) returns (GetPartitionResponse); + // List chunks in a partition + rpc ListPartitionChunks(ListPartitionChunksRequest) returns (ListPartitionChunksResponse); + // Create a new chunk in the mutable buffer rpc NewPartitionChunk(NewPartitionChunkRequest) returns (NewPartitionChunkResponse); @@ -142,8 +145,8 @@ message ListPartitionsResponse { repeated Partition partitions = 1; } -// Request to get details of a specific partition from a named database -message GetPartitionRequest { +// Request to list all chunks in a specific partitions from a named database +message ListPartitionChunksRequest { // the name of the database string db_name = 1; @@ -156,6 +159,20 @@ message GetPartitionResponse { Partition partition = 1; } +message ListPartitionChunksResponse { + // All chunks in a partition + repeated Chunk chunks = 1; +} + +// Request to get details of a specific partition from a named database +message GetPartitionRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; +} + // Request that a new chunk for writing is created in the mutable buffer message NewPartitionChunkRequest { // the name of the database diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index 759ae53502..43cc804d0f 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -145,6 +145,14 @@ pub enum GetPartitionError { ServerError(tonic::Status), } +/// Errors returned by Client::list_partition_chunks +#[derive(Debug, Error)] +pub enum ListPartitionChunksError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// Errors returned by Client::new_partition_chunk #[derive(Debug, Error)] pub enum NewPartitionChunkError { @@ -377,6 +385,26 @@ impl Client { partition.ok_or(GetPartitionError::PartitionNotFound) } + /// List chunks in a partition + pub async fn list_partition_chunks( + &mut self, + db_name: impl Into, + partition_key: impl Into, + ) -> Result, ListPartitionChunksError> { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + let response = self + .inner + .list_partition_chunks(ListPartitionChunksRequest { + db_name, + partition_key, + }) + .await + .map_err(ListPartitionChunksError::ServerError)?; + Ok(response.into_inner().chunks) + } + /// Create a new chunk in a partittion pub async fn new_partition_chunk( &mut self, diff --git a/server/src/db.rs b/server/src/db.rs index fc05d83c82..4ebc913761 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -269,6 +269,17 @@ impl Db { Ok(()) } + + /// Return Summary information for chunks in the specified partition + pub fn partition_chunk_summaries( + &self, + partition_key: &str, + ) -> impl Iterator { + self.mutable_buffer_chunks(&partition_key) + .into_iter() + .chain(self.read_buffer_chunks(&partition_key).into_iter()) + .map(|c| c.summary()) + } } impl PartialEq for Db { @@ -326,12 +337,7 @@ impl Database for Db { let summaries = self .partition_keys()? .into_iter() - .map(|partition_key| { - self.mutable_buffer_chunks(&partition_key) - .into_iter() - .chain(self.read_buffer_chunks(&partition_key).into_iter()) - .map(|c| c.summary()) - }) + .map(|partition_key| self.partition_chunk_summaries(&partition_key)) .flatten() .collect(); Ok(summaries) @@ -583,6 +589,47 @@ mod tests { db.rules.mutable_buffer_config = Some(mbconf); } + #[tokio::test] + async fn partition_chunk_summaries() { + // Test that chunk id listing is hooked up + let db = make_db(); + let mut writer = TestLPWriter::default(); + + writer.write_lp_string(&db, "cpu bar=1 1").await.unwrap(); + db.rollover_partition("1970-01-01T00").await.unwrap(); + + // write into a separate partitiion + writer + .write_lp_string(&db, "cpu bar=1,baz2,frob=3 400000000000000") + .await + .unwrap(); + + print!("Partitions: {:?}", db.partition_keys().unwrap()); + + fn to_arc(s: &str) -> Arc { + Arc::new(s.to_string()) + } + + let mut chunk_summaries = db + .partition_chunk_summaries("1970-01-05T15") + .collect::>(); + + chunk_summaries.sort_unstable(); + + let expected = vec![ChunkSummary { + partition_key: to_arc("1970-01-05T15"), + id: 0, + storage: ChunkStorage::OpenMutableBuffer, + estimated_bytes: 107, + }]; + + assert_eq!( + expected, chunk_summaries, + "expected:\n{:#?}\n\nactual:{:#?}\n\n", + expected, chunk_summaries + ); + } + #[tokio::test] async fn chunk_summaries() { // Test that chunk id listing is hooked up diff --git a/src/commands/database/partition.rs b/src/commands/database/partition.rs index 39c4df3f4a..9ba078d338 100644 --- a/src/commands/database/partition.rs +++ b/src/commands/database/partition.rs @@ -1,8 +1,14 @@ //! This module implements the `partition` CLI command +use data_types::chunk::ChunkSummary; +use generated_types::google::FieldViolation; use influxdb_iox_client::{ connection::Builder, - management::{self, GetPartitionError, ListPartitionsError, NewPartitionChunkError}, + management::{ + self, GetPartitionError, ListPartitionChunksError, ListPartitionsError, + NewPartitionChunkError, + }, }; +use std::convert::TryFrom; use structopt::StructOpt; use thiserror::Error; @@ -14,8 +20,14 @@ pub enum Error { #[error("Error getting partition: {0}")] GetPartitionsError(#[from] GetPartitionError), - #[error("Error getting partition: {0}")] - NewPartitionError(#[from] NewPartitionChunkError), + #[error("Error listing partition chunks: {0}")] + ListPartitionChunksError(#[from] ListPartitionChunksError), + + #[error("Error creating new partition chunk: {0}")] + NewPartitionChunkError(#[from] NewPartitionChunkError), + + #[error("Error interpreting server response: {0}")] + ConvertingResponse(#[from] FieldViolation), #[error("Error rendering response as JSON: {0}")] WritingJson(#[from] serde_json::Error), @@ -52,6 +64,16 @@ struct Get { partition_key: String, } +/// lists all chunks in this partition +#[derive(Debug, StructOpt)] +struct ListChunks { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, +} + /// Create a new, open chunk in the partiton's Mutable Buffer which will receive /// new writes. #[derive(Debug, StructOpt)] @@ -70,6 +92,8 @@ enum Command { List(List), // Get details about a particular partition Get(Get), + // List chunks in a partition + ListChunks(ListChunks), // Create a new chunk in the partition NewChunk(NewChunk), } @@ -107,6 +131,21 @@ pub async fn command(url: String, config: Config) -> Result<()> { serde_json::to_writer_pretty(std::io::stdout(), &partition_detail)?; } + Command::ListChunks(list_chunks) => { + let ListChunks { + db_name, + partition_key, + } = list_chunks; + + let chunks = client.list_partition_chunks(db_name, partition_key).await?; + + let chunks = chunks + .into_iter() + .map(ChunkSummary::try_from) + .collect::, FieldViolation>>()?; + + serde_json::to_writer_pretty(std::io::stdout(), &chunks)?; + } Command::NewChunk(new_chunk) => { let NewChunk { db_name, diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index d54b26b327..d4ddab439f 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -234,6 +234,30 @@ where Ok(Response::new(GetPartitionResponse { partition })) } + async fn list_partition_chunks( + &self, + request: Request, + ) -> Result, Status> { + let ListPartitionChunksRequest { + db_name, + partition_key, + } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + let chunks: Vec = db + .partition_chunk_summaries(&partition_key) + .map(|summary| summary.into()) + .collect(); + + Ok(Response::new(ListPartitionChunksResponse { chunks })) + } + async fn new_partition_chunk( &self, request: Request, diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index b9079a76ad..8bdcb21bf9 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -392,6 +392,63 @@ async fn test_partition_get_error() { assert_contains!(err.to_string(), "Partition not found"); } +#[tokio::test] +async fn test_list_partition_chunks() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + "disk,region=east bytes=99i 200", + ]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let partition_key = "cpu"; + let chunks = management_client + .list_partition_chunks(&db_name, partition_key) + .await + .expect("getting partition chunks"); + + let expected: Vec = vec![Chunk { + partition_key: "cpu".into(), + id: 0, + storage: ChunkStorage::OpenMutableBuffer as i32, + estimated_bytes: 145, + }]; + + assert_eq!( + expected, chunks, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, chunks + ); +} + +#[tokio::test] +async fn test_list_partition_chunk_errors() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let db_name = rand_name(); + + let err = management_client + .list_partition_chunks(&db_name, "cpu") + .await + .expect_err("no db had been created"); + + assert_contains!( + err.to_string(), + "Some requested entity was not found: Resource database" + ); +} + #[tokio::test] async fn test_new_partition_chunk() { let fixture = ServerFixture::create_shared().await; diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index a690f6ced3..4202220680 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -377,6 +377,70 @@ async fn test_get_partition_error() { .stderr(predicate::str::contains("Database not found")); } +#[tokio::test] +async fn test_list_partition_chunks() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu2,region=west user=21.0 150", + ]; + + load_lp(addr, &db_name, lp_data); + + let expected = r#" + "partition_key": "cpu", + "id": 0, + "storage": "OpenMutableBuffer", +"#; + + let partition_key = "cpu"; + // should not contain anything related to cpu2 partition + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list-chunks") + .arg(&db_name) + .arg(&partition_key) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected).and(predicate::str::contains("cpu2").not())); +} + +#[tokio::test] +async fn test_list_partition_chunks_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + // note don't make the database, expect error + + // list the chunks + let partition_key = "cpu"; + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list-chunks") + .arg(&db_name) + .arg(&partition_key) + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr( + predicate::str::contains("Some requested entity was not found: Resource database") + .and(predicate::str::contains(&db_name)), + ); +} + #[tokio::test] async fn test_new_partition_chunk() { let server_fixture = ServerFixture::create_shared().await; From 72eff5eed580a435fb5394f36321d5c8a5917026 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 16 Mar 2021 17:57:25 -0400 Subject: [PATCH 054/104] chore: update deps (including arrow) --- Cargo.lock | 174 +++++++++++++++++------------------ arrow_deps/Cargo.toml | 10 +- arrow_deps/src/test_util.rs | 2 +- ingest/src/parquet/writer.rs | 14 +-- query/src/provider.rs | 1 + 5 files changed, 99 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3d63943c0..76af77bde8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,9 +29,9 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" [[package]] name = "ahash" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd69f6bbb6851e9bba54499698ed6b50640e35544e6b552e7604ad6258778b9" +checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" dependencies = [ "getrandom 0.2.2", "once_cell", @@ -101,7 +101,7 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrow" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ "cfg_aliases", "chrono", @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ "arrow", "bytes", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.42" +version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ "proc-macro2", "quote", @@ -277,7 +277,7 @@ dependencies = [ "serde_json", "smallvec", "thiserror", - "time 0.2.25", + "time 0.2.26", "url", "uuid", ] @@ -404,9 +404,9 @@ checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -501,9 +501,9 @@ dependencies = [ [[package]] name = "cloud-storage" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bce9e4bdb24400b2e2403aecceb676c355c0309e3476a79c80c34858fddd611f" +checksum = "a27c92803d7c48c97d828f468d0bb3069f21a2531ccc8361486a7e05ba9518ec" dependencies = [ "base64 0.13.0", "bytes", @@ -714,9 +714,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", @@ -767,9 +767,9 @@ dependencies = [ [[package]] name = "datafusion" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ - "ahash 0.7.1", + "ahash 0.7.2", "arrow", "async-trait", "chrono", @@ -785,7 +785,6 @@ dependencies = [ "sqlparser", "tokio", "tokio-stream", - "unicode-segmentation", ] [[package]] @@ -1254,9 +1253,9 @@ dependencies = [ [[package]] name = "hex" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hmac" @@ -1380,9 +1379,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown", @@ -1630,9 +1629,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "538c092e5586f4cdd7dd8078c4a79220e3e168880218124dcbce860f0ea938c6" [[package]] name = "libloading" @@ -1775,9 +1774,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +checksum = "2182a122f3b7f3f5329cb1972cee089ba2459a0a80a56935e6e674f096f8d839" dependencies = [ "libc", "log", @@ -1816,9 +1815,9 @@ dependencies = [ [[package]] name = "multimap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1255076139a83bb467426e7f8d0134968a8118844faa755985e077cf31850333" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "mutable_buffer" @@ -1889,7 +1888,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7a8e9be5e039e2ff869df49155f1c06bd01ade2117ec783e56ab0932b67a8f" dependencies = [ - "num-bigint 0.3.1", + "num-bigint 0.3.2", "num-complex", "num-integer", "num-iter", @@ -1910,9 +1909,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9a41747ae4633fce5adffb4d2e81ffc5e89593cb19917f8fb2cc5ff76507bf" +checksum = "7d0a3d5e207573f948a9e5376662aa743a2ea13f7c50a554d7af443a73fbfeba" dependencies = [ "autocfg", "num-integer", @@ -1956,7 +1955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ "autocfg", - "num-bigint 0.3.1", + "num-bigint 0.3.2", "num-integer", "num-traits", ] @@ -2033,9 +2032,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.7.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10acf907b94fc1b1a152d08ef97e7759650268cf986bf127f387e602b02c7e5a" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" dependencies = [ "parking_lot", ] @@ -2054,15 +2053,15 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.32" +version = "0.10.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" dependencies = [ "bitflags", "cfg-if 1.0.0", "foreign-types", - "lazy_static", "libc", + "once_cell", "openssl-sys", ] @@ -2074,9 +2073,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.60" +version = "0.9.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" dependencies = [ "autocfg", "cc", @@ -2194,7 +2193,7 @@ dependencies = [ [[package]] name = "parquet" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ "arrow", "base64 0.12.3", @@ -2203,7 +2202,7 @@ dependencies = [ "chrono", "flate2", "lz4", - "num-bigint 0.3.1", + "num-bigint 0.3.2", "parquet-format", "snap", "thrift", @@ -2221,9 +2220,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d65c4d95931acda4498f675e332fcbdc9a06705cd07086c510e9b6009cd1c1" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" [[package]] name = "peeking_take_while" @@ -2306,9 +2305,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -2702,14 +2701,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -2724,9 +2722,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" [[package]] name = "remove_dir_all" @@ -2739,9 +2737,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0460542b551950620a3648c6aa23318ac6b3cd779114bd873209e6e8b5eb1c34" +checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4" dependencies = [ "base64 0.13.0", "bytes", @@ -2881,7 +2879,7 @@ dependencies = [ "rustc_version", "serde", "sha2", - "time 0.2.25", + "time 0.2.26", "tokio", ] @@ -2986,9 +2984,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfd318104249865096c8da1dfabf09ddbb6d0330ea176812a62ec75e40c4166" +checksum = "d493c5f39e02dfb062cd8f33301f90f9b13b650e8c1b1d0fd75c19dd64bff69d" dependencies = [ "bitflags", "core-foundation", @@ -3024,9 +3022,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f" dependencies = [ "serde_derive", ] @@ -3055,9 +3053,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ "proc-macro2", "quote", @@ -3066,9 +3064,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43535db9747a4ba938c0ce0a98cc631a46ebf943c9e1d604e091df6007620bf6" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "indexmap", "itoa", @@ -3379,9 +3377,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" dependencies = [ "proc-macro2", "quote", @@ -3517,9 +3515,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" +checksum = "08a8cbfbf47955132d0202d1662f49b2423ae35862aee471f3ba4b133358f372" dependencies = [ "const_fn", "libc", @@ -3555,9 +3553,9 @@ dependencies = [ [[package]] name = "tinytemplate" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ada8616fad06a2d0c455adc530de4ef57605a8120cc65da9653e0e9623ca74" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" dependencies = [ "serde", "serde_json", @@ -3580,9 +3578,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda" dependencies = [ "autocfg", "bytes", @@ -3632,9 +3630,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" +checksum = "c535f53c0cfa1acace62995a8994fc9cc1f12d202420da96ff306ee24d576469" dependencies = [ "futures-core", "pin-project-lite", @@ -3643,9 +3641,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29" dependencies = [ "bytes", "futures-core", @@ -3713,9 +3711,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713c629c07a3a97f741c140e474e7304294fabec66a43a33f0832e98315ab07f" +checksum = "f715efe02c0862926eb463e49368d38ddb119383475686178e32e26d15d06a66" dependencies = [ "futures-core", "futures-util", @@ -3758,9 +3756,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.13" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a9bd1db7706f2373a190b0d067146caa39350c486f3d455b0e33b431f94c07" +checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" dependencies = [ "proc-macro2", "quote", @@ -3822,9 +3820,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ab8966ac3ca27126141f7999361cc97dd6fb4b71da04c02044fa9045d98bb96" +checksum = "705096c6f83bf68ea5d357a6aa01829ddbdac531b357b45abeca842938085baa" dependencies = [ "ansi_term 0.12.1", "chrono", @@ -3857,9 +3855,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" [[package]] name = "unicode-bidi" @@ -3940,9 +3938,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "wait-timeout" @@ -4175,18 +4173,18 @@ checksum = "81a974bcdd357f0dca4d41677db03436324d45a4c9ed2d0b873a5a360ce41c36" [[package]] name = "zstd" -version = "0.6.0+zstd.1.4.8" +version = "0.6.1+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e44664feba7f2f1a9f300c1f6157f2d1bfc3c15c6f3cf4beabf3f5abe9c237" +checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "3.0.0+zstd.1.4.8" +version = "3.0.1+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9447afcd795693ad59918c7bbffe42fdd6e467d708f3537e3dc14dc598c573f" +checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c" dependencies = [ "libc", "zstd-sys", @@ -4194,12 +4192,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.4.19+zstd.1.4.8" +version = "1.4.20+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec24a9273d24437afb8e71b16f3d9a5d569193cccdb7896213b59f552f387674" +checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e" dependencies = [ "cc", - "glob", - "itertools 0.9.0", "libc", ] diff --git a/arrow_deps/Cargo.toml b/arrow_deps/Cargo.toml index 492657c5cb..ed63129754 100644 --- a/arrow_deps/Cargo.toml +++ b/arrow_deps/Cargo.toml @@ -8,14 +8,14 @@ description = "Apache Arrow / Parquet / DataFusion dependencies for InfluxDB IOx [dependencies] # In alphabetical order # We are using development version of arrow/parquet/datafusion and the dependencies are at the same rev -# The version can be found here: https://github.com/apache/arrow/commit/4f6adc700d1cebc50a0594b5aa671f64491cc20e +# The version can be found here: https://github.com/apache/arrow/commit/6208a79739d0228ecc566fa8436ee61068452212 # -arrow = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e" , features = ["simd"] } -arrow-flight = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e" } +arrow = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212" , features = ["simd"] } +arrow-flight = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212" } # Turn off optional datafusion features (function packages) -datafusion = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e", default-features = false } +datafusion = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212", default-features = false } # Turn off the "arrow" feature; it currently has a bug that causes the crate to rebuild every time # and we're not currently using it anyway -parquet = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e", default-features = false, features = ["snap", "brotli", "flate2", "lz4", "zstd"] } +parquet = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212", default-features = false, features = ["snap", "brotli", "flate2", "lz4", "zstd"] } diff --git a/arrow_deps/src/test_util.rs b/arrow_deps/src/test_util.rs index 11c17507f4..87ffb5fd4a 100644 --- a/arrow_deps/src/test_util.rs +++ b/arrow_deps/src/test_util.rs @@ -44,7 +44,7 @@ pub fn sort_record_batch(batch: RecordBatch) -> RecordBatch { }) .collect(); - let sort_output = lexsort(&sort_input).expect("Sorting to complete"); + let sort_output = lexsort(&sort_input, None).expect("Sorting to complete"); RecordBatch::try_new(batch.schema(), sort_output).unwrap() } diff --git a/ingest/src/parquet/writer.rs b/ingest/src/parquet/writer.rs index e83a7b9827..cfbfaa3f65 100644 --- a/ingest/src/parquet/writer.rs +++ b/ingest/src/parquet/writer.rs @@ -1,7 +1,7 @@ //! This module contains the code to write table data to parquet use arrow_deps::parquet::{ self, - basic::{Compression, Encoding, LogicalType, Repetition, Type as PhysicalType}, + basic::{Compression, ConvertedType, Encoding, Repetition, Type as PhysicalType}, errors::ParquetError, file::{ properties::{WriterProperties, WriterPropertiesBuilder}, @@ -297,19 +297,19 @@ fn convert_to_parquet_schema(schema: &Schema) -> Result (PhysicalType::BYTE_ARRAY, Some(LogicalType::UTF8)), + Some(InfluxColumnType::Tag) => (PhysicalType::BYTE_ARRAY, Some(ConvertedType::UTF8)), Some(InfluxColumnType::Field(InfluxFieldType::Boolean)) => { (PhysicalType::BOOLEAN, None) } Some(InfluxColumnType::Field(InfluxFieldType::Float)) => (PhysicalType::DOUBLE, None), Some(InfluxColumnType::Field(InfluxFieldType::Integer)) => { - (PhysicalType::INT64, Some(LogicalType::UINT_64)) + (PhysicalType::INT64, Some(ConvertedType::UINT_64)) } Some(InfluxColumnType::Field(InfluxFieldType::UInteger)) => { - (PhysicalType::INT64, Some(LogicalType::UINT_64)) + (PhysicalType::INT64, Some(ConvertedType::UINT_64)) } Some(InfluxColumnType::Field(InfluxFieldType::String)) => { - (PhysicalType::BYTE_ARRAY, Some(LogicalType::UTF8)) + (PhysicalType::BYTE_ARRAY, Some(ConvertedType::UTF8)) } Some(InfluxColumnType::Timestamp) => { // At the time of writing, the underlying rust parquet @@ -325,7 +325,7 @@ fn convert_to_parquet_schema(schema: &Schema) -> Result { return UnsupportedDataType { @@ -340,7 +340,7 @@ fn convert_to_parquet_schema(schema: &Schema) -> Result TableProvider for ChunkTableProvider { projection: &Option>, _batch_size: usize, _filters: &[Expr], + _limit: Option, ) -> std::result::Result, DataFusionError> { // TODO Here is where predicate pushdown will happen. To make // predicate push down happen, the provider need need to From 2f77090ecf9ac7ddfcbf0d9b95dd65798cad9d40 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 16 Mar 2021 18:16:48 -0400 Subject: [PATCH 055/104] fix: clippy/fmt --- server/src/db/chunk.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/db/chunk.rs b/server/src/db/chunk.rs index 377edf546f..9713c8d83f 100644 --- a/server/src/db/chunk.rs +++ b/server/src/db/chunk.rs @@ -249,10 +249,10 @@ impl PartitionChunk for DBChunk { selection: Selection<'_>, ) -> Result { match self { - DBChunk::MutableBuffer { chunk, .. } => chunk + Self::MutableBuffer { chunk, .. } => chunk .table_schema(table_name, selection) .context(MutableBufferChunk), - DBChunk::ReadBuffer { + Self::ReadBuffer { db, partition_key, chunk_id, @@ -284,7 +284,7 @@ impl PartitionChunk for DBChunk { Ok(schema) } - DBChunk::ParquetFile => { + Self::ParquetFile => { unimplemented!("parquet file not implemented for table schema") } } From 4b98d19d70f2600e084245d41249a7340f1dccae Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Wed, 17 Mar 2021 12:58:27 +0000 Subject: [PATCH 056/104] refactor: remove interior mutability from TrackerRegistry (#1004) * refactor: remove interior mutability from TrackerRegistry * fix: don't hold mutex across await point Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- server/src/lib.rs | 20 ++++++++++++-------- server/src/tracker.rs | 28 ++++++++++++++-------------- server/src/tracker/registry.rs | 28 ++++++++++++---------------- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/server/src/lib.rs b/server/src/lib.rs index 8b3c7969f0..630503d92e 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -75,6 +75,7 @@ use std::sync::{ use async_trait::async_trait; use bytes::Bytes; use futures::stream::TryStreamExt; +use parking_lot::Mutex; use snafu::{OptionExt, ResultExt, Snafu}; use tracing::{error, info}; @@ -158,7 +159,7 @@ pub struct Server { connection_manager: Arc, pub store: Arc, executor: Arc, - jobs: TrackerRegistry, + jobs: Mutex>, } impl Server { @@ -169,7 +170,7 @@ impl Server { store, connection_manager: Arc::new(connection_manager), executor: Arc::new(Executor::new()), - jobs: TrackerRegistry::new(), + jobs: Mutex::new(TrackerRegistry::new()), } } @@ -350,7 +351,7 @@ impl Server { let writer_id = self.require_id()?; let store = Arc::clone(&self.store); - let (_, tracker) = self.jobs.register(Job::PersistSegment { + let (_, tracker) = self.jobs.lock().register(Job::PersistSegment { writer_id, segment_id: segment.id, }); @@ -386,7 +387,7 @@ impl Server { } pub fn spawn_dummy_job(&self, nanos: Vec) -> Tracker { - let (tracker, registration) = self.jobs.register(Job::Dummy { + let (tracker, registration) = self.jobs.lock().register(Job::Dummy { nanos: nanos.clone(), }); @@ -402,12 +403,12 @@ impl Server { /// Returns a list of all jobs tracked by this server pub fn tracked_jobs(&self) -> Vec> { - self.jobs.tracked() + self.jobs.lock().tracked() } /// Returns a specific job tracked by this server pub fn get_job(&self, id: TrackerId) -> Option> { - self.jobs.get(id) + self.jobs.lock().get(id) } /// Background worker function @@ -419,12 +420,15 @@ impl Server { loop { // TODO: Retain limited history of past jobs, e.g. enqueue returned data into a // Dequeue - let reclaimed = self.jobs.reclaim(); + let mut jobs = self.jobs.lock(); - for job in reclaimed { + for job in jobs.reclaim() { info!(?job, "job finished"); } + // Ensure mutex guard is not held across await point + std::mem::drop(jobs); + interval.tick().await; } } diff --git a/server/src/tracker.rs b/server/src/tracker.rs index 53493a91ba..7fd8c3d457 100644 --- a/server/src/tracker.rs +++ b/server/src/tracker.rs @@ -274,7 +274,7 @@ mod tests { #[tokio::test] async fn test_lifecycle() { let (sender, receive) = oneshot::channel(); - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration) = registry.register(()); let task = tokio::spawn(receive.track(registration)); @@ -291,7 +291,7 @@ mod tests { async fn test_interleaved() { let (sender1, receive1) = oneshot::channel(); let (sender2, receive2) = oneshot::channel(); - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration1) = registry.register(1); let (_, registration2) = registry.register(2); @@ -316,7 +316,7 @@ mod tests { #[tokio::test] async fn test_drop() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration) = registry.register(()); { @@ -332,7 +332,7 @@ mod tests { #[tokio::test] async fn test_drop_multiple() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration) = registry.register(()); { @@ -351,7 +351,7 @@ mod tests { #[tokio::test] async fn test_terminate() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration) = registry.register(()); let task = tokio::spawn(futures::future::pending::<()>().track(registration)); @@ -368,7 +368,7 @@ mod tests { #[tokio::test] async fn test_terminate_early() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (tracker, registration) = registry.register(()); tracker.cancel(); @@ -381,7 +381,7 @@ mod tests { #[tokio::test] async fn test_terminate_multiple() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration) = registry.register(()); let task1 = tokio::spawn(futures::future::pending::<()>().track(registration.clone())); @@ -402,7 +402,7 @@ mod tests { #[tokio::test] async fn test_reclaim() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration1) = registry.register(1); let (_, registration2) = registry.register(2); @@ -440,7 +440,7 @@ mod tests { assert_eq!(get_metadata(&tracked), vec![1, 2, 3]); // Expect reclaim to find now finished registration1 - let reclaimed = sorted(registry.reclaim()); + let reclaimed = sorted(registry.reclaim().collect()); assert_eq!(reclaimed.len(), 1); assert_eq!(get_metadata(&reclaimed), vec![1]); @@ -485,7 +485,7 @@ mod tests { assert_eq!(running[0].created_futures(), 2); assert!(running[0].is_complete()); - let reclaimed = sorted(registry.reclaim()); + let reclaimed = sorted(registry.reclaim().collect()); assert_eq!(reclaimed.len(), 2); assert_eq!(get_metadata(&reclaimed), vec![2, 3]); @@ -496,7 +496,7 @@ mod tests { // to prevent stalling the tokio executor #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_timing() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (tracker1, registration1) = registry.register(1); let (tracker2, registration2) = registry.register(2); let (tracker3, registration3) = registry.register(3); @@ -562,20 +562,20 @@ mod tests { #[tokio::test] async fn test_register_race() { - let registry = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); let (_, registration) = registry.register(()); let task1 = tokio::spawn(futures::future::ready(()).track(registration.clone())); task1.await.unwrap().unwrap(); // Should only consider tasks complete once cannot register more Futures - let reclaimed = registry.reclaim(); + let reclaimed: Vec<_> = registry.reclaim().collect(); assert_eq!(reclaimed.len(), 0); let task2 = tokio::spawn(futures::future::ready(()).track(registration)); task2.await.unwrap().unwrap(); - let reclaimed = registry.reclaim(); + let reclaimed: Vec<_> = registry.reclaim().collect(); assert_eq!(reclaimed.len(), 1); } diff --git a/server/src/tracker/registry.rs b/server/src/tracker/registry.rs index 88028269c6..d8716289d6 100644 --- a/server/src/tracker/registry.rs +++ b/server/src/tracker/registry.rs @@ -1,8 +1,6 @@ use super::{Tracker, TrackerRegistration}; use hashbrown::HashMap; -use parking_lot::Mutex; use std::str::FromStr; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tracing::debug; @@ -38,14 +36,14 @@ struct TrackerSlot { /// Additionally can trigger graceful cancellation of registered futures #[derive(Debug)] pub struct TrackerRegistry { - next_id: AtomicUsize, - trackers: Mutex>>, + next_id: usize, + trackers: HashMap>, } impl Default for TrackerRegistry { fn default() -> Self { Self { - next_id: AtomicUsize::new(0), + next_id: 0, trackers: Default::default(), } } @@ -57,8 +55,10 @@ impl TrackerRegistry { } /// Register a new tracker in the registry - pub fn register(&self, metadata: T) -> (Tracker, TrackerRegistration) { - let id = TrackerId(self.next_id.fetch_add(1, Ordering::Relaxed)); + pub fn register(&mut self, metadata: T) -> (Tracker, TrackerRegistration) { + let id = TrackerId(self.next_id); + self.next_id += 1; + let (sender, receiver) = tokio::sync::watch::channel(false); let registration = TrackerRegistration::new(receiver); @@ -68,7 +68,7 @@ impl TrackerRegistry { state: Arc::clone(®istration.state), }; - self.trackers.lock().insert( + self.trackers.insert( id, TrackerSlot { tracker: tracker.clone(), @@ -79,11 +79,10 @@ impl TrackerRegistry { (tracker, registration) } - /// Removes completed tasks from the registry and returns a list of those - /// removed - pub fn reclaim(&self) -> Vec> { + /// Removes completed tasks from the registry and returns an iterator of + /// those removed + pub fn reclaim(&mut self) -> impl Iterator> + '_ { self.trackers - .lock() .drain_filter(|_, v| v.tracker.is_complete()) .map(|(_, v)| { if let Err(error) = v.watch.send(true) { @@ -92,19 +91,17 @@ impl TrackerRegistry { } v.tracker }) - .collect() } } impl TrackerRegistry { pub fn get(&self, id: TrackerId) -> Option> { - self.trackers.lock().get(&id).map(|x| x.tracker.clone()) + self.trackers.get(&id).map(|x| x.tracker.clone()) } /// Returns a list of trackers, including those that are no longer running pub fn tracked(&self) -> Vec> { self.trackers - .lock() .iter() .map(|(_, v)| v.tracker.clone()) .collect() @@ -113,7 +110,6 @@ impl TrackerRegistry { /// Returns a list of active trackers pub fn running(&self) -> Vec> { self.trackers - .lock() .iter() .filter_map(|(_, v)| { if !v.tracker.is_complete() { From 1fee56274b195dcb67a0d5b1442f125725cf526b Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Wed, 17 Mar 2021 14:28:54 +0000 Subject: [PATCH 057/104] fix: prevent observing inconsistent tracker states (#994) * fix: prevent observing inconsistent tracker states * feat: further tracker docs Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- server/src/tracker.rs | 159 ++++++++++++++++++---------- server/src/tracker/future.rs | 4 + src/influxdb_ioxd/rpc/operations.rs | 55 ++++++++-- 3 files changed, 149 insertions(+), 69 deletions(-) diff --git a/server/src/tracker.rs b/server/src/tracker.rs index 7fd8c3d457..a1daaa4c3d 100644 --- a/server/src/tracker.rs +++ b/server/src/tracker.rs @@ -19,8 +19,8 @@ //! //! # Correctness //! -//! The key correctness property of the Tracker system is Tracker::is_complete -//! only returns true when all futures associated with the tracker have +//! The key correctness property of the Tracker system is Tracker::get_status +//! only returns Complete when all futures associated with the tracker have //! completed and no more can be spawned. Additionally at such a point //! all metrics - cpu_nanos, wall_nanos, created_futures should be visible //! to the calling thread @@ -62,10 +62,10 @@ //! 9. 6. + 8. -> A thread that observes a pending_registrations of 0 cannot //! subsequently observe pending_futures to increase //! -//! 10. Tracker::is_complete returns if it observes pending_registrations to be -//! 0 and then pending_futures to be 0 +//! 10. Tracker::get_status returns Complete if it observes +//! pending_registrations to be 0 and then pending_futures to be 0 //! -//! 11. 9 + 10 -> A thread can only observe Tracker::is_complete() == true +//! 11. 9 + 10 -> A thread can only observe a tracker to be complete //! after all futures have been dropped and no more can be created //! //! 12. pending_futures is decremented with Release semantics on @@ -108,6 +108,41 @@ struct TrackerState { watch: tokio::sync::watch::Receiver, } +/// The status of the tracker +#[derive(Debug, Clone)] +pub enum TrackerStatus { + /// More futures can be registered + Creating, + + /// No more futures can be registered + /// + /// `pending_count` and `cpu_nanos` are best-effort - + /// they may not be the absolute latest values. + /// + /// `total_count` is guaranteed to be the final value + Running { + /// The number of created futures + total_count: usize, + /// The number of pending futures + pending_count: usize, + /// The total amount of CPU time spent executing the futures + cpu_nanos: usize, + }, + + /// All futures have been dropped and no more can be registered + /// + /// All values are guaranteed to be the final values + Complete { + /// The number of created futures + total_count: usize, + /// The total amount of CPU time spent executing the futures + cpu_nanos: usize, + /// The number of nanoseconds between tracker registration and + /// the last TrackedFuture being dropped + wall_nanos: usize, + }, +} + /// A Tracker can be used to monitor/cancel/wait for a set of associated futures #[derive(Debug)] pub struct Tracker { @@ -147,36 +182,14 @@ impl Tracker { self.state.cancel_token.cancel(); } - /// Returns the number of outstanding futures - pub fn pending_futures(&self) -> usize { - self.state.pending_futures.load(Ordering::Relaxed) - } - - /// Returns the number of TrackedFutures created with this Tracker - pub fn created_futures(&self) -> usize { - self.state.created_futures.load(Ordering::Relaxed) - } - - /// Returns the number of nanoseconds futures tracked by this - /// tracker have spent executing - pub fn cpu_nanos(&self) -> usize { - self.state.cpu_nanos.load(Ordering::Relaxed) - } - - /// Returns the number of nanoseconds since the Tracker was registered - /// to the time the last TrackedFuture was dropped - /// - /// Returns 0 if there are still pending tasks - pub fn wall_nanos(&self) -> usize { - if !self.is_complete() { - return 0; - } - self.state.wall_nanos.load(Ordering::Relaxed) - } - /// Returns true if all futures associated with this tracker have /// been dropped and no more can be created pub fn is_complete(&self) -> bool { + matches!(self.get_status(), TrackerStatus::Complete{..}) + } + + /// Gets the status of the tracker + pub fn get_status(&self) -> TrackerStatus { // The atomic decrement in TrackerRegistration::drop has release semantics // acquire here ensures that if a thread observes the tracker to have // no pending_registrations it cannot subsequently observe pending_futures @@ -189,7 +202,19 @@ impl Tracker { // a TrackedFuture, it is guaranteed to see its updates (e.g. wall_nanos) let pending_futures = self.state.pending_futures.load(Ordering::Acquire); - pending_registrations == 0 && pending_futures == 0 + match (pending_registrations == 0, pending_futures == 0) { + (false, _) => TrackerStatus::Creating, + (true, false) => TrackerStatus::Running { + total_count: self.state.created_futures.load(Ordering::Relaxed), + pending_count: self.state.pending_futures.load(Ordering::Relaxed), + cpu_nanos: self.state.cpu_nanos.load(Ordering::Relaxed), + }, + (true, true) => TrackerStatus::Complete { + total_count: self.state.created_futures.load(Ordering::Relaxed), + cpu_nanos: self.state.cpu_nanos.load(Ordering::Relaxed), + wall_nanos: self.state.wall_nanos.load(Ordering::Relaxed), + }, + } } /// Returns if this tracker has been cancelled @@ -256,10 +281,14 @@ impl TrackerRegistration { impl Drop for TrackerRegistration { fn drop(&mut self) { + // This synchronizes with the Acquire load in Tracker::get_status let previous = self .state .pending_registrations .fetch_sub(1, Ordering::Release); + + // This implies a TrackerRegistration has been cloned without it incrementing + // the pending_registration counter assert_ne!(previous, 0); } } @@ -457,9 +486,9 @@ mod tests { let result3 = task3.await.unwrap(); assert!(result3.is_ok()); - assert_eq!(tracked[0].pending_futures(), 1); - assert_eq!(tracked[0].created_futures(), 2); - assert!(!tracked[0].is_complete()); + assert!( + matches!(tracked[0].get_status(), TrackerStatus::Running { pending_count: 1, total_count: 2, ..}) + ); // Trigger termination of task5 running[1].cancel(); @@ -480,10 +509,7 @@ mod tests { let result4 = task4.await.unwrap(); assert!(result4.is_err()); - - assert_eq!(running[0].pending_futures(), 0); - assert_eq!(running[0].created_futures(), 2); - assert!(running[0].is_complete()); + assert!(matches!(running[0].get_status(), TrackerStatus::Complete { total_count: 2, ..})); let reclaimed = sorted(registry.reclaim().collect()); @@ -521,18 +547,6 @@ mod tests { task3.await.unwrap().unwrap(); task4.await.unwrap().unwrap(); - assert_eq!(tracker1.pending_futures(), 0); - assert_eq!(tracker2.pending_futures(), 0); - assert_eq!(tracker3.pending_futures(), 0); - - assert!(tracker1.is_complete()); - assert!(tracker2.is_complete()); - assert!(tracker3.is_complete()); - - assert_eq!(tracker2.created_futures(), 1); - assert_eq!(tracker2.created_futures(), 1); - assert_eq!(tracker3.created_futures(), 2); - let assert_fuzzy = |actual: usize, expected: std::time::Duration| { // Number of milliseconds of toleration let epsilon = Duration::from_millis(10).as_nanos() as usize; @@ -552,12 +566,37 @@ mod tests { ); }; - assert_fuzzy(tracker1.cpu_nanos(), Duration::from_millis(0)); - assert_fuzzy(tracker1.wall_nanos(), Duration::from_millis(100)); - assert_fuzzy(tracker2.cpu_nanos(), Duration::from_millis(100)); - assert_fuzzy(tracker2.wall_nanos(), Duration::from_millis(100)); - assert_fuzzy(tracker3.cpu_nanos(), Duration::from_millis(200)); - assert_fuzzy(tracker3.wall_nanos(), Duration::from_millis(100)); + let assert_complete = |status: TrackerStatus, + expected_cpu: std::time::Duration, + expected_wal: std::time::Duration| { + match status { + TrackerStatus::Complete { + cpu_nanos, + wall_nanos, + .. + } => { + assert_fuzzy(cpu_nanos, expected_cpu); + assert_fuzzy(wall_nanos, expected_wal); + } + _ => panic!("expected complete got {:?}", status), + } + }; + + assert_complete( + tracker1.get_status(), + Duration::from_millis(0), + Duration::from_millis(100), + ); + assert_complete( + tracker2.get_status(), + Duration::from_millis(100), + Duration::from_millis(100), + ); + assert_complete( + tracker3.get_status(), + Duration::from_millis(200), + Duration::from_millis(100), + ); } #[tokio::test] @@ -568,6 +607,10 @@ mod tests { let task1 = tokio::spawn(futures::future::ready(()).track(registration.clone())); task1.await.unwrap().unwrap(); + let tracked = registry.tracked(); + assert_eq!(tracked.len(), 1); + assert!(matches!(&tracked[0].get_status(), TrackerStatus::Creating)); + // Should only consider tasks complete once cannot register more Futures let reclaimed: Vec<_> = registry.reclaim().collect(); assert_eq!(reclaimed.len(), 0); diff --git a/server/src/tracker/future.rs b/server/src/tracker/future.rs index 342a89adb2..1a28734bf7 100644 --- a/server/src/tracker/future.rs +++ b/server/src/tracker/future.rs @@ -81,7 +81,11 @@ impl PinnedDrop for TrackedFuture { state.wall_nanos.fetch_max(wall_nanos, Ordering::Relaxed); + // This synchronizes with the Acquire load in Tracker::get_status let previous = state.pending_futures.fetch_sub(1, Ordering::Release); + + // Failure implies a TrackedFuture has somehow been created + // without it incrementing the pending_futures counter assert_ne!(previous, 0); } } diff --git a/src/influxdb_ioxd/rpc/operations.rs b/src/influxdb_ioxd/rpc/operations.rs index 871e020ff2..298dae08a5 100644 --- a/src/influxdb_ioxd/rpc/operations.rs +++ b/src/influxdb_ioxd/rpc/operations.rs @@ -19,7 +19,7 @@ use generated_types::{ protobuf_type_url, }; use server::{ - tracker::{Tracker, TrackerId}, + tracker::{Tracker, TrackerId, TrackerStatus}, ConnectionManager, Server, }; use std::convert::TryInto; @@ -31,19 +31,52 @@ struct OperationsService { pub fn encode_tracker(tracker: Tracker) -> Result { let id = tracker.id(); - let is_complete = tracker.is_complete(); let is_cancelled = tracker.is_cancelled(); + let status = tracker.get_status(); + + let (operation_metadata, is_complete) = match status { + TrackerStatus::Creating => { + let metadata = management::OperationMetadata { + job: Some(tracker.metadata().clone().into()), + ..Default::default() + }; + + (metadata, false) + } + TrackerStatus::Running { + total_count, + pending_count, + cpu_nanos, + } => { + let metadata = management::OperationMetadata { + cpu_nanos: cpu_nanos as _, + task_count: total_count as _, + pending_count: pending_count as _, + job: Some(tracker.metadata().clone().into()), + ..Default::default() + }; + + (metadata, false) + } + TrackerStatus::Complete { + total_count, + cpu_nanos, + wall_nanos, + } => { + let metadata = management::OperationMetadata { + cpu_nanos: cpu_nanos as _, + task_count: total_count as _, + wall_nanos: wall_nanos as _, + job: Some(tracker.metadata().clone().into()), + ..Default::default() + }; + + (metadata, true) + } + }; let mut buffer = BytesMut::new(); - management::OperationMetadata { - cpu_nanos: tracker.cpu_nanos() as _, - wall_nanos: tracker.wall_nanos() as _, - task_count: tracker.created_futures() as _, - pending_count: tracker.pending_futures() as _, - job: Some(tracker.metadata().clone().into()), - } - .encode(&mut buffer) - .map_err(|error| { + operation_metadata.encode(&mut buffer).map_err(|error| { debug!(?error, "Unexpected error"); InternalError {} })?; From 3a53923684127f2848075b8b3f63f02a5b540d6d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 17 Mar 2021 11:25:27 -0400 Subject: [PATCH 058/104] feat: Management API + CLI command to close a chunk and move to read buffer (#1002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Management API + CLI command to close a chunk and move to read buffer * refactor: Less copy-pasta * fix: track only once, use `let _` instead of `.ok()` * docs: Apply suggestions from code review fix comments ( 🤦‍♀️ for copy/pasta) Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> * docs: Update server/src/lib.rs Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> * refactor: Use DatabaseName rather than impl Into * fix: Fixup logical merge conflicts Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- data_types/src/job.rs | 40 +++- .../influxdata/iox/management/v1/jobs.proto | 28 ++- .../iox/management/v1/service.proto | 20 ++ influxdb_iox_client/src/client/management.rs | 48 +++++ server/src/db.rs | 71 ++++++- server/src/lib.rs | 176 ++++++++++++++++-- src/commands/database/partition.rs | 46 ++++- src/influxdb_ioxd/rpc/error.rs | 7 +- src/influxdb_ioxd/rpc/management.rs | 27 ++- tests/end_to_end_cases/management_api.rs | 100 +++++++++- tests/end_to_end_cases/management_cli.rs | 64 +++++++ tests/end_to_end_cases/operations_api.rs | 4 +- 12 files changed, 589 insertions(+), 42 deletions(-) diff --git a/data_types/src/job.rs b/data_types/src/job.rs index 9540fb6762..f071f907bf 100644 --- a/data_types/src/job.rs +++ b/data_types/src/job.rs @@ -9,15 +9,30 @@ use std::convert::TryFrom; /// Used in combination with TrackerRegistry /// /// TODO: Serde is temporary until prost adds JSON support -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Job { - PersistSegment { writer_id: u32, segment_id: u64 }, - Dummy { nanos: Vec }, + Dummy { + nanos: Vec, + }, + + /// Persist a WAL segment to object store + PersistSegment { + writer_id: u32, + segment_id: u64, + }, + + /// Move a chunk from mutable buffer to read buffer + CloseChunk { + db_name: String, + partition_key: String, + chunk_id: u32, + }, } impl From for management::operation_metadata::Job { fn from(job: Job) -> Self { match job { + Job::Dummy { nanos } => Self::Dummy(management::Dummy { nanos }), Job::PersistSegment { writer_id, segment_id, @@ -25,7 +40,15 @@ impl From for management::operation_metadata::Job { writer_id, segment_id, }), - Job::Dummy { nanos } => Self::Dummy(management::Dummy { nanos }), + Job::CloseChunk { + db_name, + partition_key, + chunk_id, + } => Self::CloseChunk(management::CloseChunk { + db_name, + partition_key, + chunk_id, + }), } } } @@ -42,6 +65,15 @@ impl From for Job { writer_id, segment_id, }, + Job::CloseChunk(management::CloseChunk { + db_name, + partition_key, + chunk_id, + }) => Self::CloseChunk { + db_name, + partition_key, + chunk_id, + }, } } } diff --git a/generated_types/protos/influxdata/iox/management/v1/jobs.proto b/generated_types/protos/influxdata/iox/management/v1/jobs.proto index 01691dbfad..a04ecc259f 100644 --- a/generated_types/protos/influxdata/iox/management/v1/jobs.proto +++ b/generated_types/protos/influxdata/iox/management/v1/jobs.proto @@ -2,22 +2,46 @@ syntax = "proto3"; package influxdata.iox.management.v1; message OperationMetadata { + // How many nanoseconds of CPU time have been spent on this job so far? uint64 cpu_nanos = 1; + + // How many nanoseconds has it been since the job was submitted uint64 wall_nanos = 2; + + // How many total tasks does this job have currently uint64 task_count = 3; + + // How many tasks for this job are still pending uint64 pending_count = 4; + // What kind of job is it? oneof job { Dummy dummy = 5; PersistSegment persist_segment = 6; + CloseChunk close_chunk = 7; } } +// A job that simply sleeps for a specified time and then returns success +message Dummy { + // How long the job should sleep for before returning + repeated uint64 nanos = 1; +} + +// A job that persists a WAL segment to object store message PersistSegment { uint32 writer_id = 1; uint64 segment_id = 2; } -message Dummy { - repeated uint64 nanos = 1; +// Move a chunk from mutable buffer to read buffer +message CloseChunk { + // name of the database + string db_name = 1; + + // partition key + string partition_key = 2; + + // chunk_id + uint32 chunk_id = 3; } diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index d53b225fc7..d02999bad3 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -50,6 +50,9 @@ service ManagementService { // Create a new chunk in the mutable buffer rpc NewPartitionChunk(NewPartitionChunkRequest) returns (NewPartitionChunkResponse); + // Close a chunk and move it to the read buffer + rpc ClosePartitionChunk(ClosePartitionChunkRequest) returns (ClosePartitionChunkResponse); + } message GetWriterIdRequest {} @@ -184,3 +187,20 @@ message NewPartitionChunkRequest { message NewPartitionChunkResponse { } + +// Request that a chunk be closed and moved to the read buffer +message ClosePartitionChunkRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; + + // the chunk id + uint32 chunk_id = 3; +} + +message ClosePartitionChunkResponse { + // The operation that tracks the work for migrating the chunk + google.longrunning.Operation operation = 1; +} diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index 43cc804d0f..0946609271 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -165,6 +165,22 @@ pub enum NewPartitionChunkError { ServerError(tonic::Status), } +/// Errors returned by Client::close_partition_chunk +#[derive(Debug, Error)] +pub enum ClosePartitionChunkError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Response contained no payload + #[error("Server returned an empty response")] + EmptyResponse, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// An IOx Management API client. /// /// This client wraps the underlying `tonic` generated client with a @@ -446,4 +462,36 @@ impl Client { .operation .ok_or(CreateDummyJobError::EmptyResponse)?) } + + /// Closes the specified chunk in the specified partition and + /// begins it moving to the read buffer. + /// + /// Returns the job tracking the data's movement + pub async fn close_partition_chunk( + &mut self, + db_name: impl Into, + partition_key: impl Into, + chunk_id: u32, + ) -> Result { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + let response = self + .inner + .close_partition_chunk(ClosePartitionChunkRequest { + db_name, + partition_key, + chunk_id, + }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => ClosePartitionChunkError::DatabaseNotFound, + _ => ClosePartitionChunkError::ServerError(status), + })?; + + Ok(response + .into_inner() + .operation + .ok_or(ClosePartitionChunkError::EmptyResponse)?) + } } diff --git a/server/src/db.rs b/server/src/db.rs index 4ebc913761..f557f97520 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -45,6 +45,16 @@ pub enum Error { #[snafu(display("Cannot read to this database: no mutable buffer configured"))] DatabaseNotReadable {}, + #[snafu(display( + "Only closed chunks can be moved to read buffer. Chunk {} {} was open", + partition_key, + chunk_id + ))] + ChunkNotClosed { + partition_key: String, + chunk_id: u32, + }, + #[snafu(display("Error dropping data from mutable buffer: {}", source))] MutableBufferDrop { source: mutable_buffer::database::Error, @@ -130,6 +140,16 @@ impl Db { } } + /// Return true if the specified chunk is still open for new writes + pub fn is_open_chunk(&self, partition_key: &str, chunk_id: u32) -> bool { + if let Some(mutable_buffer) = self.mutable_buffer.as_ref() { + let open_chunk_id = mutable_buffer.open_chunk_id(partition_key); + open_chunk_id == chunk_id + } else { + false + } + } + // Return a list of all chunks in the mutable_buffer (that can // potentially be migrated into the read buffer or object store) pub fn mutable_buffer_chunks(&self, partition_key: &str) -> Vec> { @@ -150,6 +170,19 @@ impl Db { chunks } + // Return the specified chunk in the mutable buffer + pub fn mutable_buffer_chunk( + &self, + partition_key: &str, + chunk_id: u32, + ) -> Result> { + self.mutable_buffer + .as_ref() + .context(DatatbaseNotWriteable)? + .get_chunk(partition_key, chunk_id) + .context(UnknownMutableBufferChunk { chunk_id }) + } + /// List chunks that are currently in the read buffer pub fn read_buffer_chunks(&self, partition_key: &str) -> Vec> { self.read_buffer @@ -210,12 +243,16 @@ impl Db { partition_key: &str, chunk_id: u32, ) -> Result> { - let mb_chunk = self - .mutable_buffer - .as_ref() - .context(DatatbaseNotWriteable)? - .get_chunk(partition_key, chunk_id) - .context(UnknownMutableBufferChunk { chunk_id })?; + let mb_chunk = self.mutable_buffer_chunk(partition_key, chunk_id)?; + + // Can't load an open chunk to the read buffer + if self.is_open_chunk(partition_key, chunk_id) { + return ChunkNotClosed { + partition_key, + chunk_id, + } + .fail(); + } let mut batches = Vec::new(); for stats in mb_chunk.table_stats().unwrap() { @@ -439,6 +476,28 @@ mod tests { assert_table_eq!(&expected, &batches); } + #[tokio::test] + async fn no_load_open_chunk() { + // Test that data can not be loaded into the ReadBuffer while + // still open (no way to ensure that new data gets into the + // read buffer) + let db = make_db(); + let mut writer = TestLPWriter::default(); + writer.write_lp_string(&db, "cpu bar=1 10").await.unwrap(); + + let partition_key = "1970-01-01T00"; + let err = db + .load_chunk_to_read_buffer(partition_key, 0) + .await + .unwrap_err(); + + // it should be the same chunk! + assert_contains!( + err.to_string(), + "Only closed chunks can be moved to read buffer. Chunk 1970-01-01T00 0 was open" + ); + } + #[tokio::test] async fn read_from_read_buffer() { // Test that data can be loaded into the ReadBuffer diff --git a/server/src/lib.rs b/server/src/lib.rs index 630503d92e..797af17685 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -77,7 +77,7 @@ use bytes::Bytes; use futures::stream::TryStreamExt; use parking_lot::Mutex; use snafu::{OptionExt, ResultExt, Snafu}; -use tracing::{error, info}; +use tracing::{debug, error, info}; use data_types::{ data::{lines_to_replicated_write, ReplicatedWrite}, @@ -122,6 +122,8 @@ pub enum Error { InvalidDatabaseName { source: DatabaseNameError }, #[snafu(display("database error: {}", source))] UnknownDatabaseError { source: DatabaseError }, + #[snafu(display("getting mutable buffer chunk: {}", source))] + MutableBufferChunk { source: DatabaseError }, #[snafu(display("no local buffer for database: {}", db))] NoLocalBuffer { db: String }, #[snafu(display("unable to get connection to remote server: {}", server))] @@ -401,6 +403,67 @@ impl Server { tracker } + /// Closes a chunk and starts moving its data to the read buffer, as a + /// background job, dropping when complete. + pub fn close_chunk( + &self, + db_name: DatabaseName<'_>, + partition_key: impl Into, + chunk_id: u32, + ) -> Result> { + let db_name = db_name.to_string(); + let name = DatabaseName::new(&db_name).context(InvalidDatabaseName)?; + + let partition_key = partition_key.into(); + + let db = self + .config + .db(&name) + .context(DatabaseNotFound { db_name: &db_name })?; + + let (tracker, registration) = self.jobs.lock().register(Job::CloseChunk { + db_name: db_name.clone(), + partition_key: partition_key.clone(), + chunk_id, + }); + + let task = async move { + // Close the chunk if it isn't already closed + if db.is_open_chunk(&partition_key, chunk_id) { + debug!(%db_name, %partition_key, %chunk_id, "Rolling over partition to close chunk"); + let result = db.rollover_partition(&partition_key).await; + + if let Err(e) = result { + info!(?e, %db_name, %partition_key, %chunk_id, "background task error during chunk closing"); + return Err(e); + } + } + + debug!(%db_name, %partition_key, %chunk_id, "background task loading chunk to read buffer"); + let result = db.load_chunk_to_read_buffer(&partition_key, chunk_id).await; + if let Err(e) = result { + info!(?e, %db_name, %partition_key, %chunk_id, "background task error loading read buffer chunk"); + return Err(e); + } + + // now, drop the chunk + debug!(%db_name, %partition_key, %chunk_id, "background task dropping mutable buffer chunk"); + let result = db.drop_mutable_buffer_chunk(&partition_key, chunk_id).await; + if let Err(e) = result { + info!(?e, %db_name, %partition_key, %chunk_id, "background task error loading read buffer chunk"); + return Err(e); + } + + debug!(%db_name, %partition_key, %chunk_id, "background task completed closing chunk"); + + Ok(()) + }; + + tokio::spawn(task.track(registration)); + + Ok(tracker) + } + /// Returns a list of all jobs tracked by this server pub fn tracked_jobs(&self) -> Vec> { self.jobs.lock().tracked() @@ -418,20 +481,18 @@ impl Server { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); loop { - // TODO: Retain limited history of past jobs, e.g. enqueue returned data into a - // Dequeue - let mut jobs = self.jobs.lock(); - - for job in jobs.reclaim() { - info!(?job, "job finished"); - } - - // Ensure mutex guard is not held across await point - std::mem::drop(jobs); - + self.reclaim_jobs(); interval.tick().await; } } + + fn reclaim_jobs(&self) { + let mut jobs = self.jobs.lock(); + + for job in jobs.reclaim() { + info!(?job, "job finished"); + } + } } #[async_trait] @@ -571,7 +632,7 @@ mod tests { }; use influxdb_line_protocol::parse_lines; use object_store::{memory::InMemory, path::ObjectStorePath}; - use query::frontend::sql::SQLQueryPlanner; + use query::{frontend::sql::SQLQueryPlanner, Database}; use crate::buffer::Segment; @@ -769,6 +830,71 @@ mod tests { Ok(()) } + #[tokio::test] + async fn close_chunk() -> Result { + test_helpers::maybe_start_logging(); + let manager = TestConnectionManager::new(); + let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); + let server = Arc::new(Server::new(manager, store)); + + let captured_server = Arc::clone(&server); + let background_handle = + tokio::task::spawn(async move { captured_server.background_worker().await }); + + server.set_id(1); + + let db_name = DatabaseName::new("foo").unwrap(); + server + .create_database(db_name.as_str(), DatabaseRules::new()) + .await?; + + let line = "cpu bar=1 10"; + let lines: Vec<_> = parse_lines(line).map(|l| l.unwrap()).collect(); + server.write_lines(&db_name, &lines).await.unwrap(); + + // start the close (note this is not an async) + let partition_key = ""; + let db_name_string = db_name.to_string(); + let tracker = server.close_chunk(db_name, partition_key, 0).unwrap(); + + let metadata = tracker.metadata(); + let expected_metadata = Job::CloseChunk { + db_name: db_name_string, + partition_key: partition_key.to_string(), + chunk_id: 0, + }; + assert_eq!(metadata, &expected_metadata); + + // wait for the job to complete + tracker.join().await; + + // Data should be in the read buffer and not in mutable buffer + let db_name = DatabaseName::new("foo").unwrap(); + let db = server.db(&db_name).unwrap(); + + let mut chunk_summaries = db.chunk_summaries().unwrap(); + chunk_summaries.sort_unstable(); + + let actual = chunk_summaries + .into_iter() + .map(|s| format!("{:?} {}", s.storage, s.id)) + .collect::>(); + + let expected = vec!["ReadBuffer 0", "OpenMutableBuffer 1"]; + + assert_eq!( + expected, actual, + "expected:\n{:#?}\n\nactual:{:#?}\n\n", + expected, actual + ); + + // ensure that we don't leave the server instance hanging around + background_handle.abort(); + let _ = background_handle.await; + + Ok(()) + } + #[tokio::test] async fn segment_persisted_on_rollover() { let manager = TestConnectionManager::new(); @@ -819,6 +945,30 @@ partition_key: assert_eq!(segment.writes[0].to_string(), write); } + #[tokio::test] + async fn background_task_cleans_jobs() -> Result { + let manager = TestConnectionManager::new(); + let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); + let server = Arc::new(Server::new(manager, store)); + let captured_server = Arc::clone(&server); + let background_handle = + tokio::task::spawn(async move { captured_server.background_worker().await }); + + let wait_nanos = 1000; + let job = server.spawn_dummy_job(vec![wait_nanos]); + + // Note: this will hang forwever if the background task has not been started + job.join().await; + + assert!(job.is_complete()); + + // ensure that we don't leave the server instance hanging around + background_handle.abort(); + let _ = background_handle.await; + + Ok(()) + } + #[derive(Snafu, Debug, Clone)] enum TestClusterError { #[snafu(display("Test cluster error: {}", message))] diff --git a/src/commands/database/partition.rs b/src/commands/database/partition.rs index 9ba078d338..47c2cb6dcb 100644 --- a/src/commands/database/partition.rs +++ b/src/commands/database/partition.rs @@ -1,14 +1,15 @@ //! This module implements the `partition` CLI command use data_types::chunk::ChunkSummary; +use data_types::job::Operation; use generated_types::google::FieldViolation; use influxdb_iox_client::{ connection::Builder, management::{ - self, GetPartitionError, ListPartitionChunksError, ListPartitionsError, - NewPartitionChunkError, + self, ClosePartitionChunkError, GetPartitionError, ListPartitionChunksError, + ListPartitionsError, NewPartitionChunkError, }, }; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use structopt::StructOpt; use thiserror::Error; @@ -26,14 +27,15 @@ pub enum Error { #[error("Error creating new partition chunk: {0}")] NewPartitionChunkError(#[from] NewPartitionChunkError), - #[error("Error interpreting server response: {0}")] - ConvertingResponse(#[from] FieldViolation), + #[error("Error closing chunk: {0}")] + ClosePartitionChunkError(#[from] ClosePartitionChunkError), #[error("Error rendering response as JSON: {0}")] WritingJson(#[from] serde_json::Error), - // #[error("Error rendering response as JSON: {0}")] - // WritingJson(#[from] serde_json::Error), + #[error("Received invalid response: {0}")] + InvalidResponse(#[from] FieldViolation), + #[error("Error connecting to IOx: {0}")] ConnectionError(#[from] influxdb_iox_client::connection::Error), } @@ -85,6 +87,20 @@ struct NewChunk { partition_key: String, } +/// Closes a chunk in the mutable buffer for writing and starts its migration to +/// the read buffer +#[derive(Debug, StructOpt)] +struct CloseChunk { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, + + /// The chunk id + chunk_id: u32, +} + /// All possible subcommands for partition #[derive(Debug, StructOpt)] enum Command { @@ -96,6 +112,8 @@ enum Command { ListChunks(ListChunks), // Create a new chunk in the partition NewChunk(NewChunk), + // Close the chunk and move to read buffer + CloseChunk(CloseChunk), } pub async fn command(url: String, config: Config) -> Result<()> { @@ -156,6 +174,20 @@ pub async fn command(url: String, config: Config) -> Result<()> { client.new_partition_chunk(db_name, partition_key).await?; println!("Ok"); } + Command::CloseChunk(close_chunk) => { + let CloseChunk { + db_name, + partition_key, + chunk_id, + } = close_chunk; + + let operation: Operation = client + .close_partition_chunk(db_name, partition_key, chunk_id) + .await? + .try_into()?; + + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } } Ok(()) diff --git a/src/influxdb_ioxd/rpc/error.rs b/src/influxdb_ioxd/rpc/error.rs index e6f71d8dc3..591cf7e18a 100644 --- a/src/influxdb_ioxd/rpc/error.rs +++ b/src/influxdb_ioxd/rpc/error.rs @@ -1,4 +1,4 @@ -use generated_types::google::{InternalError, NotFound, PreconditionViolation}; +use generated_types::google::{FieldViolation, InternalError, NotFound, PreconditionViolation}; use tracing::error; /// map common `server::Error` errors to the appropriate tonic Status @@ -18,6 +18,11 @@ pub fn default_server_error_handler(error: server::Error) -> tonic::Status { ..Default::default() } .into(), + Error::InvalidDatabaseName { source } => FieldViolation { + field: "db_name".into(), + description: source.to_string(), + } + .into(), error => { error!(?error, "Unexpected error"); InternalError {}.into() diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index d4ddab439f..02646616c1 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -132,8 +132,8 @@ where request: Request, ) -> Result, Status> { let request = request.into_inner(); - let slot = self.server.spawn_dummy_job(request.nanos); - let operation = Some(super::operations::encode_tracker(slot)?); + let tracker = self.server.spawn_dummy_job(request.nanos); + let operation = Some(super::operations::encode_tracker(tracker)?); Ok(Response::new(CreateDummyJobResponse { operation })) } @@ -280,6 +280,29 @@ where Ok(Response::new(NewPartitionChunkResponse {})) } + + async fn close_partition_chunk( + &self, + request: Request, + ) -> Result, Status> { + let ClosePartitionChunkRequest { + db_name, + partition_key, + chunk_id, + } = request.into_inner(); + + // Validate that the database name is legit + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let tracker = self + .server + .close_chunk(db_name, partition_key, chunk_id) + .map_err(default_server_error_handler)?; + + let operation = Some(super::operations::encode_tracker(tracker)?); + + Ok(Response::new(ClosePartitionChunkResponse { operation })) + } } pub fn make_server( diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 8bdcb21bf9..4a8577873f 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -1,15 +1,20 @@ use std::num::NonZeroU32; -use generated_types::google::protobuf::Empty; -use generated_types::{google::protobuf::Duration, influxdata::iox::management::v1::*}; +use generated_types::{ + google::protobuf::{Duration, Empty}, + influxdata::iox::management::v1::*, +}; use influxdb_iox_client::management::CreateDatabaseError; use test_helpers::assert_contains; -use crate::common::server_fixture::ServerFixture; - -use super::scenario::{ - create_readable_database, create_two_partition_database, create_unreadable_database, rand_name, +use super::{ + operations_api::get_operation_metadata, + scenario::{ + create_readable_database, create_two_partition_database, create_unreadable_database, + rand_name, + }, }; +use crate::common::server_fixture::ServerFixture; #[tokio::test] async fn test_list_update_remotes() { @@ -539,3 +544,86 @@ async fn test_new_partition_chunk_error() { assert_contains!(err.to_string(), "Database not found"); } + +#[tokio::test] +async fn test_close_partition_chunk() { + use influxdb_iox_client::management::generated_types::operation_metadata::Job; + use influxdb_iox_client::management::generated_types::ChunkStorage; + + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + let mut operations_client = fixture.operations_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let partition_key = "cpu"; + let lp_lines = vec!["cpu,region=west user=23.2 100"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + assert_eq!(chunks.len(), 1, "Chunks: {:#?}", chunks); + assert_eq!(chunks[0].id, 0); + assert_eq!(chunks[0].storage, ChunkStorage::OpenMutableBuffer as i32); + + // Move the chunk to read buffer + let operation = management_client + .close_partition_chunk(&db_name, partition_key, 0) + .await + .expect("new partition chunk"); + + println!("Operation response is {:?}", operation); + let operation_id = operation.name.parse().expect("not an integer"); + + let meta = get_operation_metadata(operation.metadata); + + // ensure we got a legit job description back + if let Some(Job::CloseChunk(close_chunk)) = meta.job { + assert_eq!(close_chunk.db_name, db_name); + assert_eq!(close_chunk.partition_key, partition_key); + assert_eq!(close_chunk.chunk_id, 0); + } else { + panic!("unexpected job returned") + }; + + // wait for the job to be done + operations_client + .wait_operation(operation_id, Some(std::time::Duration::from_secs(1))) + .await + .expect("failed to wait operation"); + + // And now the chunk should be good + let mut chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + chunks.sort_by(|c1, c2| c1.id.cmp(&c2.id)); + + assert_eq!(chunks.len(), 2, "Chunks: {:#?}", chunks); + assert_eq!(chunks[0].id, 0); + assert_eq!(chunks[0].storage, ChunkStorage::ReadBuffer as i32); + assert_eq!(chunks[1].id, 1); + assert_eq!(chunks[1].storage, ChunkStorage::OpenMutableBuffer as i32); +} + +#[tokio::test] +async fn test_close_partition_chunk_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + + let err = management_client + .close_partition_chunk("this database does not exist", "nor_does_this_partition", 0) + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); +} diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 4202220680..5e21722513 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -1,4 +1,5 @@ use assert_cmd::Command; +use data_types::job::{Job, Operation}; use predicates::prelude::*; use test_helpers::make_temp_file; @@ -499,6 +500,69 @@ async fn test_new_partition_chunk_error() { .stderr(predicate::str::contains("Database not found")); } +#[tokio::test] +async fn test_close_partition_chunk() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec!["cpu,region=west user=23.2 100"]; + load_lp(addr, &db_name, lp_data); + + let stdout: Operation = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("close-chunk") + .arg(&db_name) + .arg("cpu") + .arg("0") + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("Expected JSON output"); + + let expected_job = Job::CloseChunk { + db_name, + partition_key: "cpu".into(), + chunk_id: 0, + }; + + assert_eq!( + Some(expected_job), + stdout.job, + "operation was {:#?}", + stdout + ); +} + +#[tokio::test] +async fn test_close_partition_chunk_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("close-chunk") + .arg("non_existent_database") + .arg("non_existent_partition") + .arg("0") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + /// Loads the specified lines into the named database fn load_lp(addr: &str, db_name: &str, lp_data: Vec<&str>) { let lp_data_file = make_temp_file(lp_data.join("\n")); diff --git a/tests/end_to_end_cases/operations_api.rs b/tests/end_to_end_cases/operations_api.rs index d1024c9ed6..0fde9ea716 100644 --- a/tests/end_to_end_cases/operations_api.rs +++ b/tests/end_to_end_cases/operations_api.rs @@ -3,7 +3,9 @@ use generated_types::google::protobuf::Any; use influxdb_iox_client::{management::generated_types::*, operations, protobuf_type_url_eq}; use std::time::Duration; -fn get_operation_metadata(metadata: Option) -> OperationMetadata { +// TODO remove after #1001 and use something directly in the influxdb_iox_client +// crate +pub fn get_operation_metadata(metadata: Option) -> OperationMetadata { assert!(metadata.is_some()); let metadata = metadata.unwrap(); assert!(protobuf_type_url_eq(&metadata.type_url, OPERATION_METADATA)); From dd94a33bc72c5ffd9d61baac4ad8b7032707820e Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Wed, 17 Mar 2021 16:32:34 +0000 Subject: [PATCH 059/104] feat: retain limited tracker history (#1005) --- Cargo.lock | 1 + data_types/Cargo.toml | 7 +- data_types/src/job.rs | 35 +++- server/src/lib.rs | 18 +- server/src/tracker.rs | 2 + server/src/tracker/history.rs | 200 +++++++++++++++++++++++ server/src/tracker/registry.rs | 4 +- tests/end_to_end_cases/operations_cli.rs | 34 +++- 8 files changed, 275 insertions(+), 26 deletions(-) create mode 100644 server/src/tracker/history.rs diff --git a/Cargo.lock b/Cargo.lock index 76af77bde8..e0b05e9ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,6 +761,7 @@ dependencies = [ "serde_regex", "snafu", "test_helpers", + "tonic", "tracing", ] diff --git a/data_types/Cargo.toml b/data_types/Cargo.toml index 96dcb83659..f30aba1fc5 100644 --- a/data_types/Cargo.toml +++ b/data_types/Cargo.toml @@ -13,11 +13,12 @@ generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } percent-encoding = "2.1.0" prost = "0.7" -serde = "1.0" -snafu = "0.6" -tracing = "0.1" regex = "1.4" +serde = "1.0" serde_regex = "1.1" +snafu = "0.6" +tonic = { version = "0.4.0" } +tracing = "0.1" [dev-dependencies] # In alphabetical order criterion = "0.3" diff --git a/data_types/src/job.rs b/data_types/src/job.rs index f071f907bf..0a308bdda9 100644 --- a/data_types/src/job.rs +++ b/data_types/src/job.rs @@ -9,7 +9,7 @@ use std::convert::TryFrom; /// Used in combination with TrackerRegistry /// /// TODO: Serde is temporary until prost adds JSON support -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] pub enum Job { Dummy { nanos: Vec, @@ -78,6 +78,24 @@ impl From for Job { } } +/// The status of a running operation +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] +pub enum OperationStatus { + /// A task associated with the operation is running + Running, + /// All tasks associated with the operation have finished + /// + /// Note: This does not indicate success or failure only that + /// no tasks associated with the operation are running + Complete, + /// The operation was cancelled and no associated tasks are running + Cancelled, + /// An operation error was returned + /// + /// Note: The tracker system currently will never return this + Errored, +} + /// A group of asynchronous tasks being performed by an IOx server /// /// TODO: Temporary until prost adds JSON support @@ -95,6 +113,8 @@ pub struct Operation { pub cpu_time: std::time::Duration, /// Additional job metadata pub job: Option, + /// The status of the running operation + pub status: OperationStatus, } impl TryFrom for Operation { @@ -115,6 +135,18 @@ impl TryFrom for Operation { let meta: management::OperationMetadata = prost::Message::decode(metadata.value).field("metadata.value")?; + let status = match &operation.result { + None => OperationStatus::Running, + Some(longrunning::operation::Result::Response(_)) => OperationStatus::Complete, + Some(longrunning::operation::Result::Error(status)) => { + if status.code == tonic::Code::Cancelled as i32 { + OperationStatus::Cancelled + } else { + OperationStatus::Errored + } + } + }; + Ok(Self { id: operation.name.parse().field("name")?, task_count: meta.task_count, @@ -122,6 +154,7 @@ impl TryFrom for Operation { wall_time: std::time::Duration::from_nanos(meta.wall_nanos), cpu_time: std::time::Duration::from_nanos(meta.cpu_nanos), job: meta.job.map(Into::into), + status, }) } } diff --git a/server/src/lib.rs b/server/src/lib.rs index 797af17685..a5f062742f 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -89,13 +89,12 @@ use influxdb_line_protocol::ParsedLine; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use query::{exec::Executor, DatabaseStore}; -use crate::tracker::TrackedFutureExt; use crate::{ config::{ object_store_path_for_database_config, Config, GRPCConnectionString, DB_RULES_FILE_NAME, }, db::Db, - tracker::{Tracker, TrackerId, TrackerRegistry}, + tracker::{TrackedFutureExt, Tracker, TrackerId, TrackerRegistryWithHistory}, }; pub mod buffer; @@ -150,6 +149,7 @@ pub enum Error { pub type Result = std::result::Result; const STORE_ERROR_PAUSE_SECONDS: u64 = 100; +const JOB_HISTORY_SIZE: usize = 1000; /// `Server` is the container struct for how servers store data internally, as /// well as how they communicate with other servers. Each server will have one @@ -161,7 +161,7 @@ pub struct Server { connection_manager: Arc, pub store: Arc, executor: Arc, - jobs: Mutex>, + jobs: Mutex>, } impl Server { @@ -172,7 +172,7 @@ impl Server { store, connection_manager: Arc::new(connection_manager), executor: Arc::new(Executor::new()), - jobs: Mutex::new(TrackerRegistry::new()), + jobs: Mutex::new(TrackerRegistryWithHistory::new(JOB_HISTORY_SIZE)), } } @@ -481,18 +481,10 @@ impl Server { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); loop { - self.reclaim_jobs(); + self.jobs.lock().reclaim(); interval.tick().await; } } - - fn reclaim_jobs(&self) { - let mut jobs = self.jobs.lock(); - - for job in jobs.reclaim() { - info!(?job, "job finished"); - } - } } #[async_trait] diff --git a/server/src/tracker.rs b/server/src/tracker.rs index a1daaa4c3d..007f98e511 100644 --- a/server/src/tracker.rs +++ b/server/src/tracker.rs @@ -88,9 +88,11 @@ use tokio_util::sync::CancellationToken; use tracing::warn; pub use future::{TrackedFuture, TrackedFutureExt}; +pub use history::TrackerRegistryWithHistory; pub use registry::{TrackerId, TrackerRegistry}; mod future; +mod history; mod registry; /// The state shared between all sibling tasks diff --git a/server/src/tracker/history.rs b/server/src/tracker/history.rs new file mode 100644 index 0000000000..29ec3b1fc3 --- /dev/null +++ b/server/src/tracker/history.rs @@ -0,0 +1,200 @@ +use super::{Tracker, TrackerId, TrackerRegistration, TrackerRegistry}; +use hashbrown::hash_map::Entry; +use hashbrown::HashMap; +use std::hash::Hash; +use tracing::info; + +/// A wrapper around a TrackerRegistry that automatically retains a history +#[derive(Debug)] +pub struct TrackerRegistryWithHistory { + registry: TrackerRegistry, + history: SizeLimitedHashMap>, +} + +impl TrackerRegistryWithHistory { + pub fn new(capacity: usize) -> Self { + Self { + history: SizeLimitedHashMap::new(capacity), + registry: TrackerRegistry::new(), + } + } + + /// Register a new tracker in the registry + pub fn register(&mut self, metadata: T) -> (Tracker, TrackerRegistration) { + self.registry.register(metadata) + } + + /// Get the tracker associated with a given id + pub fn get(&self, id: TrackerId) -> Option> { + match self.history.get(&id) { + Some(x) => Some(x.clone()), + None => self.registry.get(id), + } + } + + /// Returns a list of trackers, including those that are no longer running + pub fn tracked(&self) -> Vec> { + let mut tracked = self.registry.tracked(); + tracked.extend(self.history.values().cloned()); + tracked + } + + /// Returns a list of active trackers + pub fn running(&self) -> Vec> { + self.registry.running() + } + + /// Reclaims jobs into the historical archive + pub fn reclaim(&mut self) { + for job in self.registry.reclaim() { + info!(?job, "job finished"); + self.history.push(job.id(), job) + } + } +} + +/// A size limited hashmap that maintains a finite number +/// of key value pairs providing O(1) key lookups +/// +/// Inserts over the capacity will overwrite previous values +#[derive(Debug)] +struct SizeLimitedHashMap { + values: HashMap, + ring: Vec, + start_idx: usize, + capacity: usize, +} + +impl SizeLimitedHashMap { + pub fn new(capacity: usize) -> Self { + Self { + values: HashMap::with_capacity(capacity), + ring: Vec::with_capacity(capacity), + start_idx: 0, + capacity, + } + } + + /// Get the value associated with a specific key + pub fn get(&self, key: &K) -> Option<&V> { + self.values.get(key) + } + + /// Returns an iterator to all values stored within the ring buffer + /// + /// Note: the order is not guaranteed + pub fn values(&self) -> impl Iterator + '_ { + self.values.values() + } + + /// Push a new value into the ring buffer + /// + /// If a value with the given key already exists, it will replace the value + /// Otherwise it will add the key and value to the buffer + /// + /// If there is insufficient capacity it will drop the oldest key value pair + /// from the buffer + pub fn push(&mut self, key: K, value: V) { + if let Entry::Occupied(occupied) = self.values.entry(key) { + // If already exists - replace existing value + occupied.replace_entry(value); + + return; + } + + if self.ring.len() < self.capacity { + // Still populating the ring + assert_eq!(self.start_idx, 0); + self.ring.push(key); + self.values.insert(key, value); + + return; + } + + // Need to swap something out of the ring + let mut old = key; + std::mem::swap(&mut self.ring[self.start_idx], &mut old); + + self.start_idx += 1; + if self.start_idx == self.capacity { + self.start_idx = 0; + } + + self.values.remove(&old); + self.values.insert(key, value); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hashmap() { + let expect = |ring: &SizeLimitedHashMap, expected: &[i32]| { + let mut values: Vec<_> = ring.values().cloned().collect(); + values.sort_unstable(); + assert_eq!(&values, expected); + }; + + let mut ring = SizeLimitedHashMap::new(5); + for i in 0..=4 { + ring.push(i, i); + } + + expect(&ring, &[0, 1, 2, 3, 4]); + + // Expect rollover + ring.push(5, 5); + expect(&ring, &[1, 2, 3, 4, 5]); + + for i in 6..=9 { + ring.push(i, i); + } + expect(&ring, &[5, 6, 7, 8, 9]); + + for i in 10..=52 { + ring.push(i + 10, i); + } + expect(&ring, &[48, 49, 50, 51, 52]); + assert_eq!(*ring.get(&60).unwrap(), 50); + } + + #[test] + fn test_tracker_archive() { + let compare = |expected_ids: &[TrackerId], archive: &TrackerRegistryWithHistory| { + let mut collected: Vec<_> = archive.history.values().map(|x| x.id()).collect(); + collected.sort(); + assert_eq!(&collected, expected_ids); + }; + + let mut archive = TrackerRegistryWithHistory::new(4); + + for i in 0..=3 { + archive.register(i); + } + + archive.reclaim(); + + compare( + &[TrackerId(0), TrackerId(1), TrackerId(2), TrackerId(3)], + &archive, + ); + + for i in 4..=7 { + archive.register(i); + } + + compare( + &[TrackerId(0), TrackerId(1), TrackerId(2), TrackerId(3)], + &archive, + ); + + archive.reclaim(); + + compare( + &[TrackerId(4), TrackerId(5), TrackerId(6), TrackerId(7)], + &archive, + ); + } +} diff --git a/server/src/tracker/registry.rs b/server/src/tracker/registry.rs index d8716289d6..43ed019b0f 100644 --- a/server/src/tracker/registry.rs +++ b/server/src/tracker/registry.rs @@ -7,7 +7,7 @@ use tracing::debug; /// Every future registered with a `TrackerRegistry` is assigned a unique /// `TrackerId` #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub struct TrackerId(usize); +pub struct TrackerId(pub(super) usize); impl FromStr for TrackerId { type Err = std::num::ParseIntError; @@ -92,9 +92,7 @@ impl TrackerRegistry { v.tracker }) } -} -impl TrackerRegistry { pub fn get(&self, id: TrackerId) -> Option> { self.trackers.get(&id).map(|x| x.tracker.clone()) } diff --git a/tests/end_to_end_cases/operations_cli.rs b/tests/end_to_end_cases/operations_cli.rs index 57495d7ab9..ac802b3889 100644 --- a/tests/end_to_end_cases/operations_cli.rs +++ b/tests/end_to_end_cases/operations_cli.rs @@ -1,6 +1,6 @@ use crate::common::server_fixture::ServerFixture; use assert_cmd::Command; -use data_types::job::{Job, Operation}; +use data_types::job::{Job, Operation, OperationStatus}; use predicates::prelude::*; #[tokio::test] @@ -30,7 +30,7 @@ async fn test_start_stop() { _ => panic!("expected dummy job got {:?}", stdout.job), } - let stdout: Vec = serde_json::from_slice( + let operations: Vec = serde_json::from_slice( &Command::cargo_bin("influxdb_iox") .unwrap() .arg("operation") @@ -44,23 +44,45 @@ async fn test_start_stop() { ) .expect("expected JSON output"); - assert_eq!(stdout.len(), 1); - match &stdout[0].job { + assert_eq!(operations.len(), 1); + match &operations[0].job { Some(Job::Dummy { nanos }) => { assert_eq!(nanos.len(), 1); assert_eq!(nanos[0], duration); } - _ => panic!("expected dummy job got {:?}", &stdout[0].job), + _ => panic!("expected dummy job got {:?}", &operations[0].job), } + let id = operations[0].id; + Command::cargo_bin("influxdb_iox") .unwrap() .arg("operation") .arg("cancel") - .arg(stdout[0].id.to_string()) + .arg(id.to_string()) .arg("--host") .arg(addr) .assert() .success() .stdout(predicate::str::contains("Ok")); + + let completed: Operation = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("wait") + .arg(id.to_string()) + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("expected JSON output"); + + assert_eq!(completed.pending_count, 0); + assert_eq!(completed.task_count, 1); + assert_eq!(completed.status, OperationStatus::Cancelled); + assert_eq!(&completed.job, &operations[0].job) } From 0c383b27ac02d91e0e7404a01169a1f4569f8211 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 18 Mar 2021 08:47:57 -0400 Subject: [PATCH 060/104] chore: Improve use of logging macros in storage service (#1011) --- src/influxdb_ioxd/rpc/storage/service.rs | 79 +++++------------------- 1 file changed, 15 insertions(+), 64 deletions(-) diff --git a/src/influxdb_ioxd/rpc/storage/service.rs b/src/influxdb_ioxd/rpc/storage/service.rs index a2ec41425a..a3c1cfc24d 100644 --- a/src/influxdb_ioxd/rpc/storage/service.rs +++ b/src/influxdb_ioxd/rpc/storage/service.rs @@ -226,12 +226,7 @@ where predicate, } = read_filter_request; - info!( - "read_filter for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(),"read filter"); read_filter_impl( tx.clone(), @@ -268,11 +263,7 @@ where hints, } = read_group_request; - info!( - "read_group for database {}, range: {:?}, group_keys: {:?}, group: {:?}, aggregate: {:?}, predicate: {}", - db_name, range, group_keys, group, aggregate, - predicate.loggable() - ); + info!(%db_name, ?range, ?group_keys, ?group, ?aggregate,predicate=%predicate.loggable(),"read_group"); if hints != 0 { InternalHintsFieldNotSupported { hints }.fail()? @@ -326,11 +317,7 @@ where window, } = read_window_aggregate_request; - info!( - "read_window_aggregate for database {}, range: {:?}, window_every: {:?}, offset: {:?}, aggregate: {:?}, window: {:?}, predicate: {}", - db_name, range, window_every, offset, aggregate, window, - predicate.loggable() - ); + info!(%db_name, ?range, ?window_every, ?offset, ?aggregate, ?window, predicate=%predicate.loggable(),"read_window_aggregate"); let aggregate_string = format!( "aggregate: {:?}, window_every: {:?}, offset: {:?}, window: {:?}", @@ -372,12 +359,7 @@ where predicate, } = tag_keys_request; - info!( - "tag_keys for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "tag_keys"); let measurement = None; @@ -422,23 +404,18 @@ where // Special case a request for 'tag_key=_measurement" means to list all // measurements let response = if tag_key.is_measurement() { - info!( - "tag_values with tag_key=[x00] (measurement name) for database {}, range: {:?}, predicate: {} --> returning measurement_names", - db_name, range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "tag_values with tag_key=[x00] (measurement name)"); if predicate.is_some() { - unimplemented!("tag_value for a measurement, with general predicate"); + return Err(Error::NotYetImplemented { + operation: "tag_value for a measurement, with general predicate".to_string(), + } + .to_status()); } measurement_name_impl(Arc::clone(&self.db_store), db_name, range).await } else if tag_key.is_field() { - info!( - "tag_values with tag_key=[xff] (field name) for database {}, range: {:?}, predicate: {} --> returning fields", - db_name, range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "tag_values with tag_key=[xff] (field name)"); let fieldlist = field_names_impl(Arc::clone(&self.db_store), db_name, None, range, predicate) @@ -455,13 +432,7 @@ where } else { let tag_key = String::from_utf8(tag_key).context(ConvertingTagKeyInTagValues)?; - info!( - "tag_values for database {}, range: {:?}, tag_key: {}, predicate: {}", - db_name, - range, - tag_key, - predicate.loggable() - ); + info!(%db_name, ?range, %tag_key, predicate=%predicate.loggable(), "tag_values",); tag_values_impl( Arc::clone(&self.db_store), @@ -557,12 +528,7 @@ where .map_err(|e| e.to_status()); } - info!( - "measurement_names for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "measurement_names"); let response = measurement_name_impl(Arc::clone(&self.db_store), db_name, range) .await @@ -594,13 +560,7 @@ where predicate, } = measurement_tag_keys_request; - info!( - "measurement_tag_keys for database {}, range: {:?}, measurement: {}, predicate: {}", - db_name, - range, - measurement, - predicate.loggable() - ); + info!(%db_name, ?range, %measurement, predicate=%predicate.loggable(), "measurement_tag_keys"); let measurement = Some(measurement); @@ -641,11 +601,7 @@ where tag_key, } = measurement_tag_values_request; - info!( - "measurement_tag_values for database {}, range: {:?}, measurement: {}, tag_key: {}, predicate: {}", - db_name, range, measurement, tag_key, - predicate.loggable() - ); + info!(%db_name, ?range, %measurement, %tag_key, predicate=%predicate.loggable(), "measurement_tag_values"); let measurement = Some(measurement); @@ -686,12 +642,7 @@ where predicate, } = measurement_fields_request; - info!( - "measurement_fields for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "measurement_fields"); let measurement = Some(measurement); From eca9ef7bdd383885c74f46e5fbb7a155227e7f84 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 19 Mar 2021 09:40:39 -0400 Subject: [PATCH 061/104] fix: remove unneded cpu feature check --- Cargo.toml | 4 --- src/cpu_feature_check/main.rs | 67 ----------------------------------- 2 files changed, 71 deletions(-) delete mode 100644 src/cpu_feature_check/main.rs diff --git a/Cargo.toml b/Cargo.toml index c576c002b4..254381aea0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,10 +108,6 @@ rand = "0.8.3" reqwest = "0.11" tempfile = "3.1.0" -[[bin]] -name = "cpu_feature_check" -path = "src/cpu_feature_check/main.rs" - [[bench]] name = "encoders" harness = false diff --git a/src/cpu_feature_check/main.rs b/src/cpu_feature_check/main.rs deleted file mode 100644 index d4b152d7c7..0000000000 --- a/src/cpu_feature_check/main.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! This program prints what available x86 features are available on this -//! processor - -macro_rules! check_feature { - ($name: tt) => { - println!(" {:10}: {}", $name, std::is_x86_feature_detected!($name)) - }; -} - -fn main() { - println!("Available CPU features on this machine:"); - - // The list of possibilities was taken from - // https://doc.rust-lang.org/reference/attributes/codegen.html#the-target_feature-attribute - // - // Features that are commented out are experimental - check_feature!("aes"); - check_feature!("pclmulqdq"); - check_feature!("rdrand"); - check_feature!("rdseed"); - check_feature!("tsc"); - check_feature!("mmx"); - check_feature!("sse"); - check_feature!("sse2"); - check_feature!("sse3"); - check_feature!("ssse3"); - check_feature!("sse4.1"); - check_feature!("sse4.2"); - check_feature!("sse4a"); - check_feature!("sha"); - check_feature!("avx"); - check_feature!("avx2"); - check_feature!("avx512f"); - check_feature!("avx512cd"); - check_feature!("avx512er"); - check_feature!("avx512pf"); - check_feature!("avx512bw"); - check_feature!("avx512dq"); - check_feature!("avx512vl"); - //check_feature!("avx512ifma"); - // check_feature!("avx512vbmi"); - // check_feature!("avx512vpopcntdq"); - // check_feature!("avx512vbmi2"); - // check_feature!("avx512gfni"); - // check_feature!("avx512vaes"); - // check_feature!("avx512vpclmulqdq"); - // check_feature!("avx512vnni"); - // check_feature!("avx512bitalg"); - // check_feature!("avx512bf16"); - // check_feature!("avx512vp2intersect"); - //check_feature!("f16c"); - check_feature!("fma"); - check_feature!("bmi1"); - check_feature!("bmi2"); - check_feature!("abm"); - check_feature!("lzcnt"); - check_feature!("tbm"); - check_feature!("popcnt"); - check_feature!("fxsr"); - check_feature!("xsave"); - check_feature!("xsaveopt"); - check_feature!("xsaves"); - check_feature!("xsavec"); - //check_feature!("cmpxchg16b"); - check_feature!("adx"); - //check_feature!("rtm"); -} From 7e6c6d67b46bdae24f055dd6fbc1856c331007df Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Fri, 19 Mar 2021 15:36:49 +0000 Subject: [PATCH 062/104] feat: graceful shutdown (#827) (#1018) * feat: graceful shutdown (#827) * chore: additional docs * chore: further docs Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- Cargo.lock | 1 + Cargo.toml | 3 +- server/src/lib.rs | 38 ++++++++----- src/influxdb_ioxd.rs | 109 ++++++++++++++++++++++++++++++++------ src/influxdb_ioxd/http.rs | 29 ++++++---- src/influxdb_ioxd/rpc.rs | 21 +++----- 6 files changed, 145 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0b05e9ad6..f96426c444 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1458,6 +1458,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", + "tokio-util", "tonic", "tonic-health", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 254381aea0..5b70053332 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,8 +81,9 @@ serde_urlencoded = "0.7.0" snafu = "0.6.9" structopt = "0.3.21" thiserror = "1.0.23" -tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "parking_lot"] } +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "parking_lot", "signal"] } tokio-stream = { version = "0.1.2", features = ["net"] } +tokio-util = { version = "0.6.3" } tonic = "0.4.0" tonic-health = "0.3.0" tracing = { version = "0.1", features = ["release_max_level_debug"] } diff --git a/server/src/lib.rs b/server/src/lib.rs index a5f062742f..197bc72659 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -475,14 +475,16 @@ impl Server { } /// Background worker function - /// - /// TOOD: Handle termination (#827) - pub async fn background_worker(&self) { + pub async fn background_worker(&self, shutdown: tokio_util::sync::CancellationToken) { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); - loop { + while !shutdown.is_cancelled() { self.jobs.lock().reclaim(); - interval.tick().await; + + tokio::select! { + _ = interval.tick() => {}, + _ = shutdown.cancelled() => break + } } } } @@ -629,6 +631,8 @@ mod tests { use crate::buffer::Segment; use super::*; + use tokio::task::JoinHandle; + use tokio_util::sync::CancellationToken; type TestError = Box; type Result = std::result::Result; @@ -829,9 +833,8 @@ mod tests { let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); let server = Arc::new(Server::new(manager, store)); - let captured_server = Arc::clone(&server); - let background_handle = - tokio::task::spawn(async move { captured_server.background_worker().await }); + let cancel_token = CancellationToken::new(); + let background_handle = spawn_worker(Arc::clone(&server), cancel_token.clone()); server.set_id(1); @@ -881,7 +884,7 @@ mod tests { ); // ensure that we don't leave the server instance hanging around - background_handle.abort(); + cancel_token.cancel(); let _ = background_handle.await; Ok(()) @@ -942,20 +945,20 @@ partition_key: let manager = TestConnectionManager::new(); let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); let server = Arc::new(Server::new(manager, store)); - let captured_server = Arc::clone(&server); - let background_handle = - tokio::task::spawn(async move { captured_server.background_worker().await }); + + let cancel_token = CancellationToken::new(); + let background_handle = spawn_worker(Arc::clone(&server), cancel_token.clone()); let wait_nanos = 1000; let job = server.spawn_dummy_job(vec![wait_nanos]); - // Note: this will hang forwever if the background task has not been started + // Note: this will hang forever if the background task has not been started job.join().await; assert!(job.is_complete()); // ensure that we don't leave the server instance hanging around - background_handle.abort(); + cancel_token.cancel(); let _ = background_handle.await; Ok(()) @@ -1015,4 +1018,11 @@ partition_key: fn parsed_lines(lp: &str) -> Vec> { parse_lines(lp).map(|l| l.unwrap()).collect() } + + fn spawn_worker(server: Arc>, token: CancellationToken) -> JoinHandle<()> + where + M: ConnectionManager + Send + Sync + 'static, + { + tokio::task::spawn(async move { server.background_worker(token).await }) + } } diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index 3ffa282aa0..1200729c0b 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -2,7 +2,8 @@ use crate::commands::{ logging::LoggingLevel, run::{Config, ObjectStore as ObjStoreOpt}, }; -use hyper::Server; +use futures::{pin_mut, FutureExt}; +use hyper::server::conn::AddrIncoming; use object_store::{ self, aws::AmazonS3, azure::MicrosoftAzure, gcp::GoogleCloudStorage, ObjectStore, }; @@ -47,7 +48,7 @@ pub enum Error { ServingHttp { source: hyper::Error }, #[snafu(display("Error serving RPC: {}", source))] - ServingRPC { source: self::rpc::Error }, + ServingRPC { source: tonic::transport::Error }, #[snafu(display( "Specified {} for the object store, required configuration missing for {}", @@ -68,6 +69,27 @@ pub enum Error { pub type Result = std::result::Result; +/// On unix platforms we want to intercept SIGINT and SIGTERM +/// This method returns if either are signalled +#[cfg(unix)] +async fn wait_for_signal() { + use tokio::signal::unix::{signal, SignalKind}; + let mut term = signal(SignalKind::terminate()).expect("failed to register signal handler"); + let mut int = signal(SignalKind::interrupt()).expect("failed to register signal handler"); + + tokio::select! { + _ = term.recv() => info!("Received SIGTERM"), + _ = int.recv() => info!("Received SIGINT"), + } +} + +#[cfg(windows)] +/// ctrl_c is the cross-platform way to intercept the equivalent of SIGINT +/// This method returns if this occurs +async fn wait_for_signal() { + let _ = tokio::signal::ctrl_c().await; +} + /// This is the entry point for the IOx server. `config` represents /// command line arguments, if any /// @@ -118,41 +140,94 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { warn!("server ID not set. ID must be set via the INFLUXDB_IOX_ID config or API before writing or querying data."); } - // Construct and start up gRPC server + // Construct a token to trigger shutdown + let token = tokio_util::sync::CancellationToken::new(); + // Construct and start up gRPC server let grpc_bind_addr = config.grpc_bind_address; let socket = tokio::net::TcpListener::bind(grpc_bind_addr) .await .context(StartListeningGrpc { grpc_bind_addr })?; - let grpc_server = self::rpc::make_server(socket, Arc::clone(&app_server)); + let grpc_server = rpc::serve(socket, Arc::clone(&app_server), token.clone()).fuse(); info!(bind_address=?grpc_bind_addr, "gRPC server listening"); - // Construct and start up HTTP server - let router_service = http::router_service(Arc::clone(&app_server)); - let bind_addr = config.http_bind_address; - let http_server = Server::try_bind(&bind_addr) - .context(StartListeningHttp { bind_addr })? - .serve(router_service); + let addr = AddrIncoming::bind(&bind_addr).context(StartListeningHttp { bind_addr })?; + + let http_server = http::serve(addr, Arc::clone(&app_server), token.clone()).fuse(); info!(bind_address=?bind_addr, "HTTP server listening"); let git_hash = option_env!("GIT_HASH").unwrap_or("UNKNOWN"); info!(git_hash, "InfluxDB IOx server ready"); // Get IOx background worker task - let app = app_server.background_worker(); + let app = app_server.background_worker(token.clone()).fuse(); - // TODO: Fix shutdown handling (#827) - let (grpc_server, server, _) = futures::future::join3(grpc_server, http_server, app).await; + // Shutdown signal + let signal = wait_for_signal().fuse(); - grpc_server.context(ServingRPC)?; - server.context(ServingHttp)?; + // There are two different select macros - tokio::select and futures::select + // + // tokio::select takes ownership of the passed future "moving" it into the + // select block. This works well when not running select inside a loop, or + // when using a future that can be dropped and recreated, often the case + // with tokio's futures e.g. `channel.recv()` + // + // futures::select is more flexible as it doesn't take ownership of the provided + // future. However, to safely provide this it imposes some additional + // requirements + // + // All passed futures must implement FusedFuture - it is IB to poll a future + // that has returned Poll::Ready(_). A FusedFuture has an is_terminated() + // method that indicates if it is safe to poll - e.g. false if it has + // returned Poll::Ready(_). futures::select uses this to implement its + // functionality. futures::FutureExt adds a fuse() method that + // wraps an arbitrary future and makes it a FusedFuture + // + // The additional requirement of futures::select is that if the future passed + // outlives the select block, it must be Unpin or already Pinned - info!("InfluxDB IOx server shutting down"); + // pin_mut constructs a Pin<&mut T> from a T by preventing moving the T + // from the current stack frame and constructing a Pin<&mut T> to it + pin_mut!(signal); + pin_mut!(app); + pin_mut!(grpc_server); + pin_mut!(http_server); - Ok(()) + // Return the first error encountered + let mut res = Ok(()); + + // Trigger graceful shutdown of server components on signal + // or any background service exiting + loop { + futures::select! { + _ = signal => info!("Shutdown requested"), + _ = app => info!("Background worker shutdown"), + result = grpc_server => match result { + Ok(_) => info!("gRPC server shutdown"), + Err(error) => { + error!(%error, "gRPC server error"); + res = res.and(Err(Error::ServingRPC{source: error})) + } + }, + result = http_server => match result { + Ok(_) => info!("HTTP server shutdown"), + Err(error) => { + error!(%error, "HTTP server error"); + res = res.and(Err(Error::ServingHttp{source: error})) + } + }, + complete => break + } + + token.cancel() + } + + info!("InfluxDB IOx server completed shutting down"); + + res } impl TryFrom<&Config> for ObjectStore { diff --git a/src/influxdb_ioxd/http.rs b/src/influxdb_ioxd/http.rs index 4b47a92608..2fdc4e65a9 100644 --- a/src/influxdb_ioxd/http.rs +++ b/src/influxdb_ioxd/http.rs @@ -34,11 +34,13 @@ use snafu::{OptionExt, ResultExt, Snafu}; use tracing::{debug, error, info}; use data_types::http::WalMetadataResponse; +use hyper::server::conn::AddrIncoming; use std::{ fmt::Debug, str::{self, FromStr}, sync::Arc, }; +use tokio_util::sync::CancellationToken; /// Constants used in API error codes. /// @@ -680,11 +682,21 @@ async fn snapshot_partition( +pub async fn serve( + addr: AddrIncoming, server: Arc>, -) -> RouterService { + shutdown: CancellationToken, +) -> Result<(), hyper::Error> +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ let router = router(server); - RouterService::new(router).unwrap() + let service = RouterService::new(router).unwrap(); + + hyper::Server::builder(addr) + .serve(service) + .with_graceful_shutdown(shutdown.cancelled()) + .await } #[cfg(test)] @@ -696,8 +708,6 @@ mod tests { use query::exec::Executor; use reqwest::{Client, Response}; - use hyper::Server; - use data_types::{ database_rules::{DatabaseRules, WalBufferConfig, WalBufferRollover}, wal::WriterSummary, @@ -1132,13 +1142,12 @@ mod tests { /// creates an instance of the http service backed by a in-memory /// testable database. Returns the url of the server fn test_server(server: Arc>) -> String { - let make_svc = router_service(server); - // NB: specify port 0 to let the OS pick the port. let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 0); - let server = Server::bind(&bind_addr).serve(make_svc); - let server_url = format!("http://{}", server.local_addr()); - tokio::task::spawn(server); + let addr = AddrIncoming::bind(&bind_addr).expect("failed to bind server"); + let server_url = format!("http://{}", addr.local_addr()); + + tokio::task::spawn(serve(addr, server, CancellationToken::new())); println!("Started server at {}", server_url); server_url } diff --git a/src/influxdb_ioxd/rpc.rs b/src/influxdb_ioxd/rpc.rs index 385071ffb9..f82c69e02f 100644 --- a/src/influxdb_ioxd/rpc.rs +++ b/src/influxdb_ioxd/rpc.rs @@ -1,12 +1,11 @@ use std::fmt::Debug; use std::sync::Arc; -use snafu::{ResultExt, Snafu}; use tokio::net::TcpListener; use tokio_stream::wrappers::TcpListenerStream; -use data_types::error::ErrorLogger; use server::{ConnectionManager, Server}; +use tokio_util::sync::CancellationToken; pub mod error; mod flight; @@ -16,19 +15,15 @@ mod storage; mod testing; mod write; -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("gRPC server error: {}", source))] - ServerError { source: tonic::transport::Error }, -} - -pub type Result = std::result::Result; - /// Instantiate a server listening on the specified address /// implementing the IOx, Storage, and Flight gRPC interfaces, the /// underlying hyper server instance. Resolves when the server has /// shutdown. -pub async fn make_server(socket: TcpListener, server: Arc>) -> Result<()> +pub async fn serve( + socket: TcpListener, + server: Arc>, + shutdown: CancellationToken, +) -> Result<(), tonic::transport::Error> where M: ConnectionManager + Send + Sync + Debug + 'static, { @@ -56,8 +51,6 @@ where .add_service(write::make_server(Arc::clone(&server))) .add_service(management::make_server(Arc::clone(&server))) .add_service(operations::make_server(server)) - .serve_with_incoming(stream) + .serve_with_incoming_shutdown(stream, shutdown.cancelled()) .await - .context(ServerError {}) - .log_if_error("Running Tonic Server") } From 6e1795fda04ce3c58fe86836cc7e196a8f4bce4c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 19 Mar 2021 12:27:57 -0400 Subject: [PATCH 063/104] refactor: Move some types (not yet exposed to clients) into internal_types (#1015) * refactor: Move some types (not yet exposed to clients) into internal_types * docs: Add README.md explaining the rationale * refactor: remove some stragglers * fix: fix benches * fix: Apply suggestions from code review Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> * fix: add clippy lints * fix: fmt * docs: Apply suggestions from code review fix typos Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- Cargo.lock | 30 ++++++++++++++----- Cargo.toml | 3 ++ benches/line_protocol_to_parquet.rs | 2 +- data_types/Cargo.toml | 10 ++----- data_types/README.md | 5 ++++ data_types/src/lib.rs | 12 -------- ingest/Cargo.toml | 2 +- ingest/src/lib.rs | 18 +++++------ ingest/src/parquet/writer.rs | 6 ++-- ingest/tests/read_write.rs | 2 +- internal_types/Cargo.toml | 27 +++++++++++++++++ internal_types/README.md | 7 +++++ .../benches/benchmark.rs | 2 +- {data_types => internal_types}/src/data.rs | 4 +-- internal_types/src/lib.rs | 11 +++++++ {data_types => internal_types}/src/schema.rs | 0 .../src/schema/builder.rs | 2 +- .../src/selection.rs | 0 mutable_buffer/Cargo.toml | 1 + mutable_buffer/src/chunk.rs | 3 +- mutable_buffer/src/column.rs | 4 +-- mutable_buffer/src/database.rs | 12 +++----- mutable_buffer/src/partition.rs | 6 ++-- mutable_buffer/src/pred.rs | 3 +- mutable_buffer/src/table.rs | 12 ++++---- packers/Cargo.toml | 2 +- packers/src/lib.rs | 2 +- packers/src/packers.rs | 2 +- query/Cargo.toml | 1 + query/src/exec/field.rs | 2 +- query/src/exec/fieldlist.rs | 2 +- query/src/frontend/influxrpc.rs | 5 ++-- query/src/frontend/sql.rs | 2 +- query/src/lib.rs | 6 ++-- query/src/plan/stringset.rs | 5 +++- query/src/predicate.rs | 3 +- query/src/provider.rs | 6 ++-- query/src/provider/physical.rs | 2 +- query/src/test.rs | 6 ++-- query/src/util.rs | 4 +-- read_buffer/Cargo.toml | 2 +- read_buffer/benches/database.rs | 2 +- read_buffer/src/chunk.rs | 2 +- read_buffer/src/lib.rs | 6 ++-- read_buffer/src/row_group.rs | 14 ++++----- read_buffer/src/schema.rs | 8 ++--- read_buffer/src/table.rs | 2 +- server/Cargo.toml | 1 + server/src/buffer.rs | 5 ++-- server/src/db.rs | 5 ++-- server/src/db/chunk.rs | 11 ++++--- server/src/db/streams.rs | 2 +- server/src/lib.rs | 3 +- server/src/query_tests/table_schema.rs | 2 +- server/src/snapshot.rs | 6 ++-- src/commands/convert.rs | 2 +- 56 files changed, 177 insertions(+), 130 deletions(-) create mode 100644 data_types/README.md create mode 100644 internal_types/Cargo.toml create mode 100644 internal_types/README.md rename {data_types => internal_types}/benches/benchmark.rs (99%) rename {data_types => internal_types}/src/data.rs (99%) create mode 100644 internal_types/src/lib.rs rename {data_types => internal_types}/src/schema.rs (100%) rename {data_types => internal_types}/src/schema/builder.rs (99%) rename {data_types => internal_types}/src/selection.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index f96426c444..661f94a1d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -747,11 +747,7 @@ dependencies = [ name = "data_types" version = "0.1.0" dependencies = [ - "arrow_deps", "chrono", - "crc32fast", - "criterion", - "flatbuffers 0.6.1", "generated_types", "influxdb_line_protocol", "percent-encoding", @@ -1429,6 +1425,7 @@ dependencies = [ "influxdb_line_protocol", "influxdb_tsm", "ingest", + "internal_types", "logfmt", "mem_qe", "mutable_buffer", @@ -1516,10 +1513,10 @@ name = "ingest" version = "0.1.0" dependencies = [ "arrow_deps", - "data_types", "flate2", "influxdb_line_protocol", "influxdb_tsm", + "internal_types", "packers", "parking_lot", "snafu", @@ -1542,6 +1539,22 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" +[[package]] +name = "internal_types" +version = "0.1.0" +dependencies = [ + "arrow_deps", + "chrono", + "crc32fast", + "criterion", + "data_types", + "flatbuffers 0.6.1", + "generated_types", + "influxdb_line_protocol", + "snafu", + "tracing", +] + [[package]] name = "ipnet" version = "2.3.0" @@ -1833,6 +1846,7 @@ dependencies = [ "flatbuffers 0.6.1", "generated_types", "influxdb_line_protocol", + "internal_types", "snafu", "string-interner", "test_helpers", @@ -2151,9 +2165,9 @@ name = "packers" version = "0.1.0" dependencies = [ "arrow_deps", - "data_types", "human_format", "influxdb_tsm", + "internal_types", "rand 0.8.3", "snafu", "test_helpers", @@ -2507,6 +2521,7 @@ dependencies = [ "data_types", "futures", "influxdb_line_protocol", + "internal_types", "parking_lot", "snafu", "sqlparser", @@ -2654,9 +2669,9 @@ dependencies = [ "arrow_deps", "criterion", "croaring", - "data_types", "either", "hashbrown", + "internal_types", "itertools 0.9.0", "packers", "permutation", @@ -3125,6 +3140,7 @@ dependencies = [ "generated_types", "hashbrown", "influxdb_line_protocol", + "internal_types", "mutable_buffer", "object_store", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index 5b70053332..89dc2b5d29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" authors = ["Paul Dix "] edition = "2018" default-run = "influxdb_iox" +readme = "README.md" [workspace] # In alphabetical order members = [ @@ -16,6 +17,7 @@ members = [ "influxdb_tsm", "influxdb2_client", "ingest", + "internal_types", "logfmt", "mem_qe", "mutable_buffer", @@ -43,6 +45,7 @@ generated_types = { path = "generated_types" } influxdb_iox_client = { path = "influxdb_iox_client", features = ["format"] } influxdb_line_protocol = { path = "influxdb_line_protocol" } influxdb_tsm = { path = "influxdb_tsm" } +internal_types = { path = "internal_types" } ingest = { path = "ingest" } logfmt = { path = "logfmt" } mem_qe = { path = "mem_qe" } diff --git a/benches/line_protocol_to_parquet.rs b/benches/line_protocol_to_parquet.rs index bc416d6980..93160bf438 100644 --- a/benches/line_protocol_to_parquet.rs +++ b/benches/line_protocol_to_parquet.rs @@ -1,5 +1,4 @@ use criterion::{criterion_group, criterion_main, Criterion, Throughput}; -use data_types::schema::Schema; use influxdb_line_protocol::parse_lines; use ingest::{ parquet::{ @@ -8,6 +7,7 @@ use ingest::{ }, ConversionSettings, LineProtocolConverter, }; +use internal_types::schema::Schema; use packers::{Error as TableError, IOxTableWriter, IOxTableWriterSource}; use std::time::Duration; diff --git a/data_types/Cargo.toml b/data_types/Cargo.toml index f30aba1fc5..8c99f52135 100644 --- a/data_types/Cargo.toml +++ b/data_types/Cargo.toml @@ -2,13 +2,12 @@ name = "data_types" version = "0.1.0" authors = ["pauldix "] +description = "InfluxDB IOx data_types, shared between IOx instances and IOx clients" edition = "2018" +readme = "README.md" [dependencies] # In alphabetical order -arrow_deps = { path = "../arrow_deps" } chrono = { version = "0.4", features = ["serde"] } -crc32fast = "1.2.0" -flatbuffers = "0.6" # TODO: Update to 0.8 generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } percent-encoding = "2.1.0" @@ -21,9 +20,4 @@ tonic = { version = "0.4.0" } tracing = "0.1" [dev-dependencies] # In alphabetical order -criterion = "0.3" test_helpers = { path = "../test_helpers" } - -[[bench]] -name = "benchmark" -harness = false diff --git a/data_types/README.md b/data_types/README.md new file mode 100644 index 0000000000..2911ec8088 --- /dev/null +++ b/data_types/README.md @@ -0,0 +1,5 @@ +# Data Types + +This crate contains types that are designed for external consumption (in `influxdb_iox_client` and other "client" facing uses). + +*Client facing* in this case means exposed via management API or CLI and where changing the structs may require additional coordination / organization with clients. diff --git a/data_types/src/lib.rs b/data_types/src/lib.rs index 574feb3c27..1d0333bf34 100644 --- a/data_types/src/lib.rs +++ b/data_types/src/lib.rs @@ -11,26 +11,14 @@ )] pub use database_name::*; -pub use schema::TIME_COLUMN_NAME; - -/// The name of the column containing table names returned by a call to -/// `table_names`. -pub const TABLE_NAMES_COLUMN_NAME: &str = "table"; - -/// The name of the column containing column names returned by a call to -/// `column_names`. -pub const COLUMN_NAMES_COLUMN_NAME: &str = "column"; pub mod chunk; -pub mod data; pub mod database_rules; pub mod error; pub mod http; pub mod job; pub mod names; pub mod partition_metadata; -pub mod schema; -pub mod selection; pub mod timestamp; pub mod wal; diff --git a/ingest/Cargo.toml b/ingest/Cargo.toml index b43ce27c94..743a283a24 100644 --- a/ingest/Cargo.toml +++ b/ingest/Cargo.toml @@ -6,9 +6,9 @@ edition = "2018" [dependencies] # In alphabetical order arrow_deps = { path = "../arrow_deps" } -data_types = { path = "../data_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } influxdb_tsm = { path = "../influxdb_tsm" } +internal_types = { path = "../internal_types" } packers = { path = "../packers" } snafu = "0.6.2" tracing = "0.1" diff --git a/ingest/src/lib.rs b/ingest/src/lib.rs index 637bdf7c58..a07b7187d5 100644 --- a/ingest/src/lib.rs +++ b/ingest/src/lib.rs @@ -11,16 +11,15 @@ clippy::clone_on_ref_ptr )] -use data_types::{ - schema::{builder::InfluxSchemaBuilder, InfluxFieldType, Schema}, - TIME_COLUMN_NAME, -}; use influxdb_line_protocol::{FieldValue, ParsedLine}; use influxdb_tsm::{ mapper::{ColumnData, MeasurementTable, TSMMeasurementMapper}, reader::{BlockDecoder, TSMBlockReader, TSMIndexReader}, BlockType, TSMError, }; +use internal_types::schema::{ + builder::InfluxSchemaBuilder, InfluxFieldType, Schema, TIME_COLUMN_NAME, +}; use packers::{ ByteArray, Error as TableError, IOxTableWriter, IOxTableWriterSource, Packer, Packers, }; @@ -75,7 +74,7 @@ pub enum Error { #[snafu(display(r#"Error building schema: {}"#, source))] BuildingSchema { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display(r#"Error writing to TableWriter: {}"#, source))] @@ -96,8 +95,8 @@ pub enum Error { CouldNotFindColumn, } -impl From for Error { - fn from(source: data_types::schema::builder::Error) -> Self { +impl From for Error { + fn from(source: internal_types::schema::builder::Error) -> Self { Self::BuildingSchema { source } } } @@ -820,7 +819,8 @@ impl TSMFileConverter { mut block_reader: impl BlockDecoder, m: &mut MeasurementTable, ) -> Result<(Schema, Vec), Error> { - let mut builder = data_types::schema::builder::SchemaBuilder::new().measurement(&m.name); + let mut builder = + internal_types::schema::builder::SchemaBuilder::new().measurement(&m.name); let mut packed_columns: Vec = Vec::new(); let mut tks = Vec::new(); @@ -1099,11 +1099,11 @@ impl std::fmt::Debug for TSMFileConverter { #[cfg(test)] mod tests { use super::*; - use data_types::{assert_column_eq, schema::InfluxColumnType}; use influxdb_tsm::{ reader::{BlockData, MockBlockDecoder}, Block, }; + use internal_types::{assert_column_eq, schema::InfluxColumnType}; use packers::{Error as TableError, IOxTableWriter, IOxTableWriterSource, Packers}; use test_helpers::approximately_equal; diff --git a/ingest/src/parquet/writer.rs b/ingest/src/parquet/writer.rs index cfbfaa3f65..ab5046fd7c 100644 --- a/ingest/src/parquet/writer.rs +++ b/ingest/src/parquet/writer.rs @@ -9,7 +9,7 @@ use arrow_deps::parquet::{ }, schema::types::{ColumnPath, Type}, }; -use data_types::schema::{InfluxColumnType, InfluxFieldType, Schema}; +use internal_types::schema::{InfluxColumnType, InfluxFieldType, Schema}; use parquet::file::writer::ParquetWriter; use snafu::{OptionExt, ResultExt, Snafu}; use std::{ @@ -97,7 +97,7 @@ where /// /// ``` /// # use std::fs; - /// # use data_types::schema::{builder::SchemaBuilder, InfluxFieldType}; + /// # use internal_types::schema::{builder::SchemaBuilder, InfluxFieldType}; /// # use packers::IOxTableWriter; /// # use packers::{Packer, Packers}; /// # use ingest::parquet::writer::{IOxParquetTableWriter, CompressionLevel}; @@ -505,7 +505,7 @@ fn create_writer_props( #[cfg(test)] mod tests { - use data_types::schema::builder::SchemaBuilder; + use internal_types::schema::builder::SchemaBuilder; use super::*; diff --git a/ingest/tests/read_write.rs b/ingest/tests/read_write.rs index 9c4f0aa78e..5ecf815291 100644 --- a/ingest/tests/read_write.rs +++ b/ingest/tests/read_write.rs @@ -1,5 +1,5 @@ -use data_types::schema::{builder::SchemaBuilder, InfluxFieldType}; use ingest::parquet::writer::{CompressionLevel, IOxParquetTableWriter}; +use internal_types::schema::{builder::SchemaBuilder, InfluxFieldType}; use packers::{IOxTableWriter, Packer, Packers}; use arrow_deps::parquet::data_type::ByteArray; diff --git a/internal_types/Cargo.toml b/internal_types/Cargo.toml new file mode 100644 index 0000000000..799f27a2d1 --- /dev/null +++ b/internal_types/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "internal_types" +version = "0.1.0" +authors = ["Andrew Lamb "] +edition = "2018" +description = "InfluxDB IOx internal types, shared between IOx instances" +readme = "README.md" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +arrow_deps = { path = "../arrow_deps" } +crc32fast = "1.2.0" +chrono = { version = "0.4", features = ["serde"] } +data_types = { path = "../data_types" } +flatbuffers = "0.6" # TODO: Update to 0.8 +generated_types = { path = "../generated_types" } +influxdb_line_protocol = { path = "../influxdb_line_protocol" } +snafu = "0.6" +tracing = "0.1" + +[dev-dependencies] # In alphabetical order +criterion = "0.3" + +[[bench]] +name = "benchmark" +harness = false diff --git a/internal_types/README.md b/internal_types/README.md new file mode 100644 index 0000000000..5ee9876ad8 --- /dev/null +++ b/internal_types/README.md @@ -0,0 +1,7 @@ +# Internal Types + +This crate contains InfluxDB IOx "internal" types which are shared +across crates and internally between IOx instances, but not exposed +externally to clients + +*Internal* in this case means that changing the structs is designed not to require additional coordination / organization with clients. diff --git a/data_types/benches/benchmark.rs b/internal_types/benches/benchmark.rs similarity index 99% rename from data_types/benches/benchmark.rs rename to internal_types/benches/benchmark.rs index 9914d41e57..7ec4db8a32 100644 --- a/data_types/benches/benchmark.rs +++ b/internal_types/benches/benchmark.rs @@ -1,9 +1,9 @@ use criterion::measurement::WallTime; use criterion::{criterion_group, criterion_main, Bencher, BenchmarkId, Criterion, Throughput}; -use data_types::data::{lines_to_replicated_write as lines_to_rw, ReplicatedWrite}; use data_types::database_rules::{DatabaseRules, PartitionTemplate, TemplatePart}; use generated_types::wal as wb; use influxdb_line_protocol::{parse_lines, ParsedLine}; +use internal_types::data::{lines_to_replicated_write as lines_to_rw, ReplicatedWrite}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::time::Duration; diff --git a/data_types/src/data.rs b/internal_types/src/data.rs similarity index 99% rename from data_types/src/data.rs rename to internal_types/src/data.rs index 20bd1c66b1..d7457d0f2e 100644 --- a/data_types/src/data.rs +++ b/internal_types/src/data.rs @@ -1,8 +1,8 @@ //! This module contains helper methods for constructing replicated writes //! based on `DatabaseRules`. -use crate::database_rules::Partitioner; -use crate::TIME_COLUMN_NAME; +use crate::schema::TIME_COLUMN_NAME; +use data_types::database_rules::Partitioner; use generated_types::wal as wb; use influxdb_line_protocol::{FieldValue, ParsedLine}; diff --git a/internal_types/src/lib.rs b/internal_types/src/lib.rs new file mode 100644 index 0000000000..ecaf7f6243 --- /dev/null +++ b/internal_types/src/lib.rs @@ -0,0 +1,11 @@ +#![deny(rust_2018_idioms)] +#![warn( + missing_debug_implementations, + clippy::explicit_iter_loop, + clippy::use_self, + clippy::clone_on_ref_ptr +)] + +pub mod data; +pub mod schema; +pub mod selection; diff --git a/data_types/src/schema.rs b/internal_types/src/schema.rs similarity index 100% rename from data_types/src/schema.rs rename to internal_types/src/schema.rs diff --git a/data_types/src/schema/builder.rs b/internal_types/src/schema/builder.rs similarity index 99% rename from data_types/src/schema/builder.rs rename to internal_types/src/schema/builder.rs index 8a04f044c5..8746d3ec93 100644 --- a/data_types/src/schema/builder.rs +++ b/internal_types/src/schema/builder.rs @@ -140,7 +140,7 @@ impl SchemaBuilder { /// schema validation happens at this time. /// ``` - /// use data_types::schema::{builder::SchemaBuilder, InfluxColumnType, InfluxFieldType}; + /// use internal_types::schema::{builder::SchemaBuilder, InfluxColumnType, InfluxFieldType}; /// /// let schema = SchemaBuilder::new() /// .tag("region") diff --git a/data_types/src/selection.rs b/internal_types/src/selection.rs similarity index 100% rename from data_types/src/selection.rs rename to internal_types/src/selection.rs diff --git a/mutable_buffer/Cargo.toml b/mutable_buffer/Cargo.toml index 638a62c2f9..efd29c249a 100644 --- a/mutable_buffer/Cargo.toml +++ b/mutable_buffer/Cargo.toml @@ -20,6 +20,7 @@ chrono = "0.4" data_types = { path = "../data_types" } flatbuffers = "0.6" # TODO: Update to 0.8 generated_types = { path = "../generated_types" } +internal_types = { path = "../internal_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } snafu = "0.6.2" string-interner = "0.12.2" diff --git a/mutable_buffer/src/chunk.rs b/mutable_buffer/src/chunk.rs index 20713c8c01..6f4ebc1a27 100644 --- a/mutable_buffer/src/chunk.rs +++ b/mutable_buffer/src/chunk.rs @@ -9,7 +9,8 @@ use chrono::{DateTime, Utc}; use generated_types::wal as wb; use std::collections::{BTreeSet, HashMap}; -use data_types::{partition_metadata::TableSummary, schema::Schema, selection::Selection}; +use data_types::partition_metadata::TableSummary; +use internal_types::{schema::Schema, selection::Selection}; use crate::{ column::Column, diff --git a/mutable_buffer/src/column.rs b/mutable_buffer/src/column.rs index d10c6e2c78..20ceafc5d3 100644 --- a/mutable_buffer/src/column.rs +++ b/mutable_buffer/src/column.rs @@ -2,9 +2,9 @@ use generated_types::wal as wb; use snafu::Snafu; use crate::dictionary::Dictionary; -use data_types::{data::type_description, partition_metadata::StatValues}; - use arrow_deps::arrow::datatypes::DataType as ArrowDataType; +use data_types::partition_metadata::StatValues; +use internal_types::data::type_description; use std::mem; diff --git a/mutable_buffer/src/database.rs b/mutable_buffer/src/database.rs index 1e095f52ff..51fe91e796 100644 --- a/mutable_buffer/src/database.rs +++ b/mutable_buffer/src/database.rs @@ -1,8 +1,6 @@ -use data_types::{ - data::ReplicatedWrite, - database_rules::{PartitionSort, PartitionSortRules}, -}; +use data_types::database_rules::{PartitionSort, PartitionSortRules}; use generated_types::wal; +use internal_types::data::ReplicatedWrite; use crate::{chunk::Chunk, partition::Partition}; @@ -249,12 +247,10 @@ impl MutableBufferDb { mod tests { use super::*; use chrono::{DateTime, Utc}; - use data_types::{ - data::lines_to_replicated_write, database_rules::Partitioner, selection::Selection, - }; + use data_types::database_rules::{Order, Partitioner}; + use internal_types::{data::lines_to_replicated_write, selection::Selection}; use arrow_deps::arrow::array::{Array, StringArray}; - use data_types::database_rules::Order; use influxdb_line_protocol::{parse_lines, ParsedLine}; type TestError = Box; diff --git a/mutable_buffer/src/partition.rs b/mutable_buffer/src/partition.rs index f050199cae..92797c92ba 100644 --- a/mutable_buffer/src/partition.rs +++ b/mutable_buffer/src/partition.rs @@ -302,10 +302,8 @@ impl<'a> Iterator for ChunkIter<'a> { mod tests { use super::*; use chrono::Utc; - use data_types::{ - data::split_lines_into_write_entry_partitions, partition_metadata::PartitionSummary, - selection::Selection, - }; + use data_types::partition_metadata::PartitionSummary; + use internal_types::{data::split_lines_into_write_entry_partitions, selection::Selection}; use arrow_deps::{ arrow::record_batch::RecordBatch, assert_table_eq, test_util::sort_record_batch, diff --git a/mutable_buffer/src/pred.rs b/mutable_buffer/src/pred.rs index 869840e302..24a7db3074 100644 --- a/mutable_buffer/src/pred.rs +++ b/mutable_buffer/src/pred.rs @@ -10,7 +10,8 @@ use arrow_deps::{ }, util::{make_range_expr, AndExprBuilder}, }; -use data_types::{timestamp::TimestampRange, TIME_COLUMN_NAME}; +use data_types::timestamp::TimestampRange; +use internal_types::schema::TIME_COLUMN_NAME; //use snafu::{OptionExt, ResultExt, Snafu}; use snafu::{ensure, ResultExt, Snafu}; diff --git a/mutable_buffer/src/table.rs b/mutable_buffer/src/table.rs index 43d275ab6f..e10d76d09f 100644 --- a/mutable_buffer/src/table.rs +++ b/mutable_buffer/src/table.rs @@ -12,12 +12,12 @@ use crate::{ dictionary::{Dictionary, Error as DictionaryError}, pred::{ChunkIdSet, ChunkPredicate}, }; -use data_types::{ - partition_metadata::{ColumnSummary, Statistics}, - schema::{builder::SchemaBuilder, Schema}, +use data_types::partition_metadata::{ColumnSummary, Statistics}; +use internal_types::{ + schema::{builder::SchemaBuilder, Schema, TIME_COLUMN_NAME}, selection::Selection, - TIME_COLUMN_NAME, }; + use snafu::{OptionExt, ResultExt, Snafu}; use arrow_deps::{ @@ -84,7 +84,7 @@ pub enum Error { #[snafu(display("Internal error converting schema: {}", source))] InternalSchema { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display( @@ -597,8 +597,8 @@ impl<'a> TableColSelection<'a> { #[cfg(test)] mod tests { - use data_types::data::split_lines_into_write_entry_partitions; use influxdb_line_protocol::{parse_lines, ParsedLine}; + use internal_types::data::split_lines_into_write_entry_partitions; use super::*; diff --git a/packers/Cargo.toml b/packers/Cargo.toml index fa9b92124c..627ac781b6 100644 --- a/packers/Cargo.toml +++ b/packers/Cargo.toml @@ -6,9 +6,9 @@ edition = "2018" [dependencies] # In alphabetical order arrow_deps = { path = "../arrow_deps" } -data_types = { path = "../data_types" } human_format = "1.0.3" influxdb_tsm = { path = "../influxdb_tsm" } +internal_types = { path = "../internal_types" } snafu = "0.6.2" tracing = "0.1" diff --git a/packers/src/lib.rs b/packers/src/lib.rs index e3197acf80..5a82ec9bdb 100644 --- a/packers/src/lib.rs +++ b/packers/src/lib.rs @@ -15,7 +15,7 @@ use snafu::Snafu; pub use crate::packers::{Packer, Packers}; pub use arrow_deps::parquet::data_type::ByteArray; -use data_types::schema::Schema; +use internal_types::schema::Schema; use std::borrow::Cow; diff --git a/packers/src/packers.rs b/packers/src/packers.rs index 59616cdf18..64d9767943 100644 --- a/packers/src/packers.rs +++ b/packers/src/packers.rs @@ -10,7 +10,7 @@ use std::iter; use std::slice::Chunks; use arrow_deps::parquet::data_type::ByteArray; -use data_types::schema::{InfluxColumnType, InfluxFieldType}; +use internal_types::schema::{InfluxColumnType, InfluxFieldType}; use std::default::Default; // NOTE: See https://blog.twitter.com/engineering/en_us/a/2013/dremel-made-simple-with-parquet.html diff --git a/query/Cargo.toml b/query/Cargo.toml index 2273117036..2276bf995d 100644 --- a/query/Cargo.toml +++ b/query/Cargo.toml @@ -21,6 +21,7 @@ croaring = "0.4.5" data_types = { path = "../data_types" } futures = "0.3.7" influxdb_line_protocol = { path = "../influxdb_line_protocol" } +internal_types = { path = "../internal_types" } parking_lot = "0.11.1" snafu = "0.6.2" sqlparser = "0.8.0" diff --git a/query/src/exec/field.rs b/query/src/exec/field.rs index a9ef1154eb..b21f42d757 100644 --- a/query/src/exec/field.rs +++ b/query/src/exec/field.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use arrow_deps::arrow::{self, datatypes::SchemaRef}; -use data_types::TIME_COLUMN_NAME; +use internal_types::schema::TIME_COLUMN_NAME; use snafu::{ResultExt, Snafu}; #[derive(Debug, Snafu)] diff --git a/query/src/exec/fieldlist.rs b/query/src/exec/fieldlist.rs index e0b846aa0c..4c30928e77 100644 --- a/query/src/exec/fieldlist.rs +++ b/query/src/exec/fieldlist.rs @@ -9,7 +9,7 @@ use arrow_deps::arrow::{ datatypes::{DataType, SchemaRef}, record_batch::RecordBatch, }; -use data_types::TIME_COLUMN_NAME; +use internal_types::schema::TIME_COLUMN_NAME; use snafu::{ensure, ResultExt, Snafu}; diff --git a/query/src/frontend/influxrpc.rs b/query/src/frontend/influxrpc.rs index 86853b4a94..590233ae8b 100644 --- a/query/src/frontend/influxrpc.rs +++ b/query/src/frontend/influxrpc.rs @@ -14,10 +14,9 @@ use arrow_deps::{ }, util::IntoExpr, }; -use data_types::{ - schema::{InfluxColumnType, Schema}, +use internal_types::{ + schema::{InfluxColumnType, Schema, TIME_COLUMN_NAME}, selection::Selection, - TIME_COLUMN_NAME, }; use snafu::{ensure, OptionExt, ResultExt, Snafu}; use tracing::debug; diff --git a/query/src/frontend/sql.rs b/query/src/frontend/sql.rs index ce305c21a0..f30788236b 100644 --- a/query/src/frontend/sql.rs +++ b/query/src/frontend/sql.rs @@ -4,7 +4,7 @@ use snafu::{ResultExt, Snafu}; use crate::{exec::Executor, provider::ProviderBuilder, Database, PartitionChunk}; use arrow_deps::datafusion::{error::DataFusionError, physical_plan::ExecutionPlan}; -use data_types::selection::Selection; +use internal_types::selection::Selection; #[derive(Debug, Snafu)] pub enum Error { diff --git a/query/src/lib.rs b/query/src/lib.rs index b1a3a53b74..446f2297f8 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -8,11 +8,9 @@ use arrow_deps::datafusion::physical_plan::SendableRecordBatchStream; use async_trait::async_trait; -use data_types::{ - chunk::ChunkSummary, data::ReplicatedWrite, partition_metadata::TableSummary, schema::Schema, - selection::Selection, -}; +use data_types::{chunk::ChunkSummary, partition_metadata::TableSummary}; use exec::{stringset::StringSet, Executor}; +use internal_types::{data::ReplicatedWrite, schema::Schema, selection::Selection}; use std::{fmt::Debug, sync::Arc}; diff --git a/query/src/plan/stringset.rs b/query/src/plan/stringset.rs index e2b9a60962..73cc850854 100644 --- a/query/src/plan/stringset.rs +++ b/query/src/plan/stringset.rs @@ -1,7 +1,10 @@ use std::sync::Arc; use arrow_deps::{datafusion::logical_plan::LogicalPlan, util::str_iter_to_batch}; -use data_types::TABLE_NAMES_COLUMN_NAME; + +/// The name of the column containing table names returned by a call to +/// `table_names`. +const TABLE_NAMES_COLUMN_NAME: &str = "table"; use crate::{ exec::stringset::{StringSet, StringSetRef}, diff --git a/query/src/predicate.rs b/query/src/predicate.rs index e2b6759ab5..d02b8dec2a 100644 --- a/query/src/predicate.rs +++ b/query/src/predicate.rs @@ -9,7 +9,8 @@ use arrow_deps::{ datafusion::logical_plan::Expr, util::{make_range_expr, AndExprBuilder}, }; -use data_types::{timestamp::TimestampRange, TIME_COLUMN_NAME}; +use data_types::timestamp::TimestampRange; +use internal_types::schema::TIME_COLUMN_NAME; /// This `Predicate` represents the empty predicate (aka that /// evaluates to true for all rows). diff --git a/query/src/provider.rs b/query/src/provider.rs index df91b53081..5a6e8065e6 100644 --- a/query/src/provider.rs +++ b/query/src/provider.rs @@ -14,7 +14,7 @@ use arrow_deps::{ physical_plan::ExecutionPlan, }, }; -use data_types::schema::{builder::SchemaMerger, Schema}; +use internal_types::schema::{builder::SchemaMerger, Schema}; use crate::{predicate::Predicate, util::project_schema, PartitionChunk}; @@ -29,7 +29,7 @@ pub enum Error { #[snafu(display("Chunk schema not compatible for table '{}': {}", table_name, source))] ChunkSchemaNotCompatible { table_name: String, - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display( @@ -39,7 +39,7 @@ pub enum Error { ))] InternalNoChunks { table_name: String, - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display("Internal error: No rows found in table '{}'", table_name))] diff --git a/query/src/provider/physical.rs b/query/src/provider/physical.rs index c17e961450..3342711d7a 100644 --- a/query/src/provider/physical.rs +++ b/query/src/provider/physical.rs @@ -9,7 +9,7 @@ use arrow_deps::{ physical_plan::{ExecutionPlan, Partitioning, SendableRecordBatchStream}, }, }; -use data_types::{schema::Schema, selection::Selection}; +use internal_types::{schema::Schema, selection::Selection}; use crate::{predicate::Predicate, PartitionChunk}; diff --git a/query/src/test.rs b/query/src/test.rs index 8117dae1df..f46740ac18 100644 --- a/query/src/test.rs +++ b/query/src/test.rs @@ -18,16 +18,16 @@ use crate::{ Database, DatabaseStore, PartitionChunk, Predicate, }; -use data_types::{ +use data_types::database_rules::{DatabaseRules, PartitionTemplate, TemplatePart}; +use influxdb_line_protocol::{parse_lines, ParsedLine}; +use internal_types::{ data::{lines_to_replicated_write, ReplicatedWrite}, - database_rules::{DatabaseRules, PartitionTemplate, TemplatePart}, schema::{ builder::{SchemaBuilder, SchemaMerger}, Schema, }, selection::Selection, }; -use influxdb_line_protocol::{parse_lines, ParsedLine}; use async_trait::async_trait; use chrono::{DateTime, Utc}; diff --git a/query/src/util.rs b/query/src/util.rs index 19244482d0..53bd71cc78 100644 --- a/query/src/util.rs +++ b/query/src/util.rs @@ -11,7 +11,7 @@ use arrow_deps::{ optimizer::utils::expr_to_column_names, }, }; -use data_types::schema::Schema; +use internal_types::schema::Schema; /// Create a logical plan that produces the record batch pub fn make_scan_plan(batch: RecordBatch) -> std::result::Result { @@ -57,7 +57,7 @@ pub fn schema_has_all_expr_columns(schema: &Schema, expr: &Expr) -> bool { #[cfg(test)] mod tests { use arrow_deps::datafusion::prelude::*; - use data_types::schema::builder::SchemaBuilder; + use internal_types::schema::builder::SchemaBuilder; use super::*; diff --git a/read_buffer/Cargo.toml b/read_buffer/Cargo.toml index de9018c428..61d37b35b0 100644 --- a/read_buffer/Cargo.toml +++ b/read_buffer/Cargo.toml @@ -13,9 +13,9 @@ edition = "2018" [dependencies] # In alphabetical order arrow_deps = { path = "../arrow_deps" } croaring = "0.4.5" -data_types = { path = "../data_types" } either = "1.6.1" hashbrown = "0.9.1" +internal_types = { path = "../internal_types" } itertools = "0.9.0" packers = { path = "../packers" } permutation = "0.2.5" diff --git a/read_buffer/benches/database.rs b/read_buffer/benches/database.rs index 1bef6a222c..975e550b1f 100644 --- a/read_buffer/benches/database.rs +++ b/read_buffer/benches/database.rs @@ -6,7 +6,7 @@ use arrow_deps::arrow::{ array::{ArrayRef, Int64Array, StringArray}, record_batch::RecordBatch, }; -use data_types::schema::builder::SchemaBuilder; +use internal_types::schema::builder::SchemaBuilder; use read_buffer::{BinaryExpr, Database, Predicate}; const BASE_TIME: i64 = 1351700038292387000_i64; diff --git a/read_buffer/src/chunk.rs b/read_buffer/src/chunk.rs index 30bccdb8eb..506b0d3338 100644 --- a/read_buffer/src/chunk.rs +++ b/read_buffer/src/chunk.rs @@ -3,7 +3,7 @@ use std::{ sync::RwLock, }; -use data_types::selection::Selection; +use internal_types::selection::Selection; use snafu::{ResultExt, Snafu}; use crate::row_group::RowGroup; diff --git a/read_buffer/src/lib.rs b/read_buffer/src/lib.rs index b0df18b8e5..af21e2d92a 100644 --- a/read_buffer/src/lib.rs +++ b/read_buffer/src/lib.rs @@ -16,7 +16,7 @@ use std::{ }; use arrow_deps::arrow::record_batch::RecordBatch; -use data_types::{ +use internal_types::{ schema::{builder::SchemaMerger, Schema}, selection::Selection, }; @@ -40,7 +40,7 @@ pub enum Error { // TODO add more context / helpful error here #[snafu(display("Error building unioned read buffer schema for chunks: {}", source))] BuildingSchema { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display("partition key does not exist: {}", key))] @@ -842,7 +842,7 @@ mod test { }, datatypes::DataType::{Boolean, Float64, Int64, UInt64, Utf8}, }; - use data_types::schema::builder::SchemaBuilder; + use internal_types::schema::builder::SchemaBuilder; use crate::value::Values; diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index 07b94fddea..d057eed6c2 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -22,13 +22,11 @@ use arrow_deps::{ arrow, datafusion::logical_plan::Expr as DfExpr, datafusion::scalar::ScalarValue as DFScalarValue, }; -use data_types::{ - schema::{InfluxColumnType, Schema}, - selection::Selection, -}; +use internal_types::schema::{InfluxColumnType, Schema}; +use internal_types::selection::Selection; /// The name used for a timestamp column. -pub const TIME_COLUMN_NAME: &str = data_types::TIME_COLUMN_NAME; +pub const TIME_COLUMN_NAME: &str = internal_types::schema::TIME_COLUMN_NAME; #[derive(Debug, Snafu)] pub enum Error { @@ -39,7 +37,7 @@ pub enum Error { #[snafu(display("schema conversion error: {}", source))] SchemaError { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display("unsupported operation: {}", msg))] @@ -1638,7 +1636,7 @@ impl TryFrom> for RecordBatch { type Error = Error; fn try_from(result: ReadFilterResult<'_>) -> Result { - let schema = data_types::schema::Schema::try_from(result.schema()) + let schema = internal_types::schema::Schema::try_from(result.schema()) .map_err(|source| Error::SchemaError { source })?; let arrow_schema: arrow_deps::arrow::datatypes::SchemaRef = schema.into(); @@ -1871,7 +1869,7 @@ impl TryFrom> for RecordBatch { type Error = Error; fn try_from(result: ReadAggregateResult<'_>) -> Result { - let schema = data_types::schema::Schema::try_from(result.schema()) + let schema = internal_types::schema::Schema::try_from(result.schema()) .map_err(|source| Error::SchemaError { source })?; let arrow_schema: arrow_deps::arrow::datatypes::SchemaRef = schema.into(); diff --git a/read_buffer/src/schema.rs b/read_buffer/src/schema.rs index 895a1b6ee7..9d9828ff2a 100644 --- a/read_buffer/src/schema.rs +++ b/read_buffer/src/schema.rs @@ -1,7 +1,7 @@ use std::{convert::TryFrom, fmt::Display}; use arrow_deps::arrow; -use data_types::schema::InfluxFieldType; +use internal_types::schema::InfluxFieldType; /// A schema that is used to track the names and semantics of columns returned /// in results out of various operations on a row group. @@ -96,11 +96,11 @@ impl Display for ResultSchema { } } -impl TryFrom<&ResultSchema> for data_types::schema::Schema { - type Error = data_types::schema::builder::Error; +impl TryFrom<&ResultSchema> for internal_types::schema::Schema { + type Error = internal_types::schema::builder::Error; fn try_from(rs: &ResultSchema) -> Result { - let mut builder = data_types::schema::builder::SchemaBuilder::new(); + let mut builder = internal_types::schema::builder::SchemaBuilder::new(); for (col_type, data_type) in &rs.select_columns { match col_type { ColumnType::Tag(name) => builder = builder.tag(name.as_str()), diff --git a/read_buffer/src/table.rs b/read_buffer/src/table.rs index e80fae23fe..f11c56e1b6 100644 --- a/read_buffer/src/table.rs +++ b/read_buffer/src/table.rs @@ -7,7 +7,7 @@ use std::{ }; use arrow_deps::arrow::record_batch::RecordBatch; -use data_types::selection::Selection; +use internal_types::selection::Selection; use snafu::{ensure, Snafu}; use crate::row_group::{self, ColumnName, GroupKey, Predicate, RowGroup}; diff --git a/server/Cargo.toml b/server/Cargo.toml index 8d26d6b735..257af8cd02 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -16,6 +16,7 @@ futures = "0.3.7" generated_types = { path = "../generated_types" } hashbrown = "0.9.1" influxdb_line_protocol = { path = "../influxdb_line_protocol" } +internal_types = { path = "../internal_types" } mutable_buffer = { path = "../mutable_buffer" } object_store = { path = "../object_store" } parking_lot = "0.11.1" diff --git a/server/src/buffer.rs b/server/src/buffer.rs index 7046cc1642..8448130ec2 100644 --- a/server/src/buffer.rs +++ b/server/src/buffer.rs @@ -1,11 +1,11 @@ //! This module contains code for managing the WAL buffer use data_types::{ - data::ReplicatedWrite, database_rules::{WalBufferRollover, WriterId}, DatabaseName, }; use generated_types::wal; +use internal_types::data::ReplicatedWrite; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use std::{ @@ -567,8 +567,9 @@ fn database_object_store_path( #[cfg(test)] mod tests { use super::*; - use data_types::{data::lines_to_replicated_write, database_rules::DatabaseRules}; + use data_types::database_rules::DatabaseRules; use influxdb_line_protocol::parse_lines; + use internal_types::data::lines_to_replicated_write; use object_store::memory::InMemory; #[test] diff --git a/server/src/db.rs b/server/src/db.rs index f557f97520..a3de6a84e3 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -10,9 +10,8 @@ use std::{ }; use async_trait::async_trait; -use data_types::{ - chunk::ChunkSummary, data::ReplicatedWrite, database_rules::DatabaseRules, selection::Selection, -}; +use data_types::{chunk::ChunkSummary, database_rules::DatabaseRules}; +use internal_types::{data::ReplicatedWrite, selection::Selection}; use mutable_buffer::MutableBufferDb; use parking_lot::Mutex; use query::{Database, PartitionChunk}; diff --git a/server/src/db/chunk.rs b/server/src/db/chunk.rs index 9713c8d83f..be1cf8747d 100644 --- a/server/src/db/chunk.rs +++ b/server/src/db/chunk.rs @@ -1,9 +1,6 @@ use arrow_deps::datafusion::physical_plan::SendableRecordBatchStream; -use data_types::{ - chunk::{ChunkStorage, ChunkSummary}, - schema::Schema, - selection::Selection, -}; +use data_types::chunk::{ChunkStorage, ChunkSummary}; +use internal_types::{schema::Schema, selection::Selection}; use mutable_buffer::chunk::Chunk as MBChunk; use query::{exec::stringset::StringSet, predicate::Predicate, PartitionChunk}; use read_buffer::Database as ReadBufferDb; @@ -33,7 +30,9 @@ pub enum Error { }, #[snafu(display("Internal error restricting schema: {}", source))] - InternalSelectingSchema { source: data_types::schema::Error }, + InternalSelectingSchema { + source: internal_types::schema::Error, + }, #[snafu(display("Predicate conversion error: {}", source))] PredicateConversion { source: super::pred::Error }, diff --git a/server/src/db/streams.rs b/server/src/db/streams.rs index c255bed513..5ea7d8cd17 100644 --- a/server/src/db/streams.rs +++ b/server/src/db/streams.rs @@ -8,7 +8,7 @@ use arrow_deps::{ }, datafusion::physical_plan::RecordBatchStream, }; -use data_types::selection::Selection; +use internal_types::selection::Selection; use mutable_buffer::chunk::Chunk as MBChunk; use read_buffer::ReadFilterResults; diff --git a/server/src/lib.rs b/server/src/lib.rs index 197bc72659..69e5dfa3b6 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -80,11 +80,12 @@ use snafu::{OptionExt, ResultExt, Snafu}; use tracing::{debug, error, info}; use data_types::{ - data::{lines_to_replicated_write, ReplicatedWrite}, database_rules::{DatabaseRules, WriterId}, job::Job, {DatabaseName, DatabaseNameError}, }; +use internal_types::data::{lines_to_replicated_write, ReplicatedWrite}; + use influxdb_line_protocol::ParsedLine; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use query::{exec::Executor, DatabaseStore}; diff --git a/server/src/query_tests/table_schema.rs b/server/src/query_tests/table_schema.rs index 9aceb80aa5..c3b49df0ae 100644 --- a/server/src/query_tests/table_schema.rs +++ b/server/src/query_tests/table_schema.rs @@ -1,7 +1,7 @@ //! Tests for the table_names implementation use arrow_deps::arrow::datatypes::DataType; -use data_types::{schema::builder::SchemaBuilder, selection::Selection}; +use internal_types::{schema::builder::SchemaBuilder, selection::Selection}; use query::{Database, PartitionChunk}; use super::scenarios::*; diff --git a/server/src/snapshot.rs b/server/src/snapshot.rs index 403435fdfa..de64050e16 100644 --- a/server/src/snapshot.rs +++ b/server/src/snapshot.rs @@ -5,10 +5,8 @@ use arrow_deps::{ datafusion::physical_plan::SendableRecordBatchStream, parquet::{self, arrow::ArrowWriter, file::writer::TryClone}, }; -use data_types::{ - partition_metadata::{PartitionSummary, TableSummary}, - selection::Selection, -}; +use data_types::partition_metadata::{PartitionSummary, TableSummary}; +use internal_types::selection::Selection; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use query::{predicate::EMPTY_PREDICATE, PartitionChunk}; diff --git a/src/commands/convert.rs b/src/commands/convert.rs index d04b5ff00f..0b64ae8f88 100644 --- a/src/commands/convert.rs +++ b/src/commands/convert.rs @@ -1,9 +1,9 @@ -use data_types::schema::Schema; use influxdb_line_protocol::parse_lines; use ingest::{ parquet::writer::{CompressionLevel, Error as ParquetWriterError, IOxParquetTableWriter}, ConversionSettings, Error as IngestError, LineProtocolConverter, TSMFileConverter, }; +use internal_types::schema::Schema; use packers::{Error as TableError, IOxTableWriter, IOxTableWriterSource}; use snafu::{OptionExt, ResultExt, Snafu}; use std::{ From 7f9bb1dbc24b2420c2856a44afe324766f253653 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Fri, 19 Mar 2021 17:37:39 +0100 Subject: [PATCH 064/104] chore: Add debuginfo=0 to CI builds --- .github/workflows/rust.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b882ff9cc0..801f78cff4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -19,6 +19,10 @@ on: [pull_request] name: ci +env: + # Disable debug symbol generation to speed up CI build + RUSTFLAGS: "-C debuginfo=0" + jobs: build: From a19c1cd3d692ec0c3f6230d8c76c9285f3e59bfd Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Fri, 19 Mar 2021 17:38:45 +0000 Subject: [PATCH 065/104] chore: install lld in CI image and remove musl (#1024) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- docker/Dockerfile.ci | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 6404c75bbf..f4f7cea9b7 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -39,8 +39,7 @@ RUN apt-get update \ && apt-get install -y \ git locales sudo openssh-client ca-certificates tar gzip parallel \ unzip zip bzip2 gnupg curl make pkg-config libssl-dev \ - musl musl-dev musl-tools clang llvm \ - jq \ + jq clang lld \ --no-install-recommends \ && apt-get clean autoclean \ && apt-get autoremove --yes \ From 13b97cc31d7d3c0d9b70143577a70aaf95cc5ff7 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Mon, 22 Mar 2021 10:13:43 +0000 Subject: [PATCH 066/104] chore: use lld on linux (#1022) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .cargo/config | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .cargo/config diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000000..9a552b1589 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,4 @@ +[target.x86_64-unknown-linux-gnu] +rustflags = [ + "-C", "link-arg=-fuse-ld=lld", +] \ No newline at end of file From aa6c21141bae98608249176ff4218b9935401280 Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Wed, 17 Mar 2021 14:13:34 -0400 Subject: [PATCH 067/104] docs: Necessary to build flatc from source now --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 75ea410db0..e33706c6b2 100644 --- a/README.md +++ b/README.md @@ -85,30 +85,30 @@ and you should see a nightly version of Rust! InfluxDB IOx uses the [FlatBuffer] serialization format for its write-ahead log. The [`flatc` compiler] reads the schema in `generated_types/wal.fbs` and generates the corresponding Rust code. -Install `flatc` >= 1.12.0 with one of these methods as appropriate to your operating system: +As of this writing (2020-03-15), we are using the `flatbuffers` crate at version 0.8.3 to match +Arrow and reduce our build times. This crate requires `flatc` to be built from source from at least +[86401e0], the same commit that changed the crate verison to 0.8.3. -* Using a [Windows binary release] -* Using the [`flatbuffers` package for conda] -* Using the [`flatbuffers` package for Arch Linux] -* Using the [`flatbuffers` package for Homebrew] +You can build `flatc` [using `cmake` as recommended in the building guide][flatc-cmake] or [using +`bazel` as Arrow does][flatc-bazel]. Make sure the resulting executable is in your `PATH`. -Once you have installed the packages, you should be able to run: +You should be able to run: ```shell flatc --version ``` -and see the version displayed. +and see version 1.12.0 displayed (even though we've built from source at a commit newer than that +released version). You won't have to run `flatc` directly; once it's available, Rust's Cargo build tool manages the compilation process by calling `flatc` for you. [FlatBuffer]: https://google.github.io/flatbuffers/ [`flatc` compiler]: https://google.github.io/flatbuffers/flatbuffers_guide_using_schema_compiler.html -[Windows binary release]: https://github.com/google/flatbuffers/releases -[`flatbuffers` package for conda]: https://anaconda.org/conda-forge/flatbuffers -[`flatbuffers` package for Arch Linux]: https://www.archlinux.org/packages/community/x86_64/flatbuffers/ -[`flatbuffers` package for Homebrew]: https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/flatbuffers.rb +[86401e0]: https://github.com/google/flatbuffers/commit/86401e078d0746d2381735415f8c2dfe849f3f52#diff-c25804a2cc9060430107b2e04f4ec904ed08b798b1bb3f8db963724f91cea522R3 +[flatc-cmake]: https://google.github.io/flatbuffers/flatbuffers_guide_building.html +[flatc-bazel]: https://github.com/apache/arrow/blob/9a80565dbb8d8abe46c1477d684eabb9ce1bb0ff/rust/arrow/regen.sh#L24-L53 ### Installing `clang` From 24af745b23ba2b91bb1215fe4a66e13248af162d Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Wed, 17 Mar 2021 14:14:43 -0400 Subject: [PATCH 068/104] chore: Upgrade to flatbuffers 0.8 --- Cargo.lock | 19 +++++-------------- generated_types/Cargo.toml | 2 +- internal_types/Cargo.toml | 2 +- mutable_buffer/Cargo.toml | 2 +- server/Cargo.toml | 2 +- 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 661f94a1d4..76c18f99f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,7 +106,7 @@ dependencies = [ "cfg_aliases", "chrono", "csv", - "flatbuffers 0.8.3", + "flatbuffers", "hex", "indexmap", "lazy_static", @@ -949,15 +949,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" -[[package]] -name = "flatbuffers" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a788f068dd10687940565bf4b5480ee943176cbd114b12e811074bcf7c04e4b9" -dependencies = [ - "smallvec", -] - [[package]] name = "flatbuffers" version = "0.8.3" @@ -1133,7 +1124,7 @@ dependencies = [ name = "generated_types" version = "0.1.0" dependencies = [ - "flatbuffers 0.6.1", + "flatbuffers", "futures", "google_types", "prost", @@ -1548,7 +1539,7 @@ dependencies = [ "crc32fast", "criterion", "data_types", - "flatbuffers 0.6.1", + "flatbuffers", "generated_types", "influxdb_line_protocol", "snafu", @@ -1843,7 +1834,7 @@ dependencies = [ "chrono", "criterion", "data_types", - "flatbuffers 0.6.1", + "flatbuffers", "generated_types", "influxdb_line_protocol", "internal_types", @@ -3135,7 +3126,7 @@ dependencies = [ "chrono", "crc32fast", "data_types", - "flatbuffers 0.6.1", + "flatbuffers", "futures", "generated_types", "hashbrown", diff --git a/generated_types/Cargo.toml b/generated_types/Cargo.toml index 7a245194cc..d99e03ff37 100644 --- a/generated_types/Cargo.toml +++ b/generated_types/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Paul Dix "] edition = "2018" [dependencies] # In alphabetical order -flatbuffers = "0.6" # TODO: Update to 0.8 +flatbuffers = "0.8" futures = "0.3.1" prost = "0.7" prost-types = "0.7" diff --git a/internal_types/Cargo.toml b/internal_types/Cargo.toml index 799f27a2d1..a0196a31dd 100644 --- a/internal_types/Cargo.toml +++ b/internal_types/Cargo.toml @@ -13,7 +13,7 @@ arrow_deps = { path = "../arrow_deps" } crc32fast = "1.2.0" chrono = { version = "0.4", features = ["serde"] } data_types = { path = "../data_types" } -flatbuffers = "0.6" # TODO: Update to 0.8 +flatbuffers = "0.8" generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } snafu = "0.6" diff --git a/mutable_buffer/Cargo.toml b/mutable_buffer/Cargo.toml index efd29c249a..a84fae9357 100644 --- a/mutable_buffer/Cargo.toml +++ b/mutable_buffer/Cargo.toml @@ -18,7 +18,7 @@ arrow_deps = { path = "../arrow_deps" } async-trait = "0.1" chrono = "0.4" data_types = { path = "../data_types" } -flatbuffers = "0.6" # TODO: Update to 0.8 +flatbuffers = "0.8" generated_types = { path = "../generated_types" } internal_types = { path = "../internal_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } diff --git a/server/Cargo.toml b/server/Cargo.toml index 257af8cd02..3e081bf725 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,7 +11,7 @@ bytes = "1.0" chrono = "0.4" crc32fast = "1.2.0" data_types = { path = "../data_types" } -flatbuffers = "0.6" # TODO: Update to 0.8 +flatbuffers = "0.8" futures = "0.3.7" generated_types = { path = "../generated_types" } hashbrown = "0.9.1" From 7c6f543d35aa74d9a192cf7b57ed3177490e9bc2 Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Wed, 17 Mar 2021 14:18:04 -0400 Subject: [PATCH 069/104] fix: wb::ColumnValue is now a set of associated consts, not a real Rust enum --- internal_types/src/data.rs | 18 +++++++++--------- mutable_buffer/src/column.rs | 14 ++++++-------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/internal_types/src/data.rs b/internal_types/src/data.rs index d7457d0f2e..28a570ae06 100644 --- a/internal_types/src/data.rs +++ b/internal_types/src/data.rs @@ -13,16 +13,15 @@ use crc32fast::Hasher; use flatbuffers::FlatBufferBuilder; pub fn type_description(value: wb::ColumnValue) -> &'static str { - use wb::ColumnValue::*; - match value { - NONE => "none", - TagValue => "tag", - I64Value => "i64", - U64Value => "u64", - F64Value => "f64", - BoolValue => "bool", - StringValue => "String", + wb::ColumnValue::TagValue => "tag", + wb::ColumnValue::I64Value => "i64", + wb::ColumnValue::U64Value => "u64", + wb::ColumnValue::F64Value => "f64", + wb::ColumnValue::BoolValue => "bool", + wb::ColumnValue::StringValue => "String", + wb::ColumnValue::NONE => "none", + _ => "none", } } @@ -143,6 +142,7 @@ impl fmt::Display for ReplicatedWrite { .unwrap_or("") .to_string(), wb::ColumnValue::NONE => "".to_string(), + _ => "".to_string(), }; write!(f, " {}:{}", value.column().unwrap_or(""), val)?; } diff --git a/mutable_buffer/src/column.rs b/mutable_buffer/src/column.rs index 20ceafc5d3..89a88e111a 100644 --- a/mutable_buffer/src/column.rs +++ b/mutable_buffer/src/column.rs @@ -46,10 +46,8 @@ impl Column { capacity: usize, value: wb::Value<'_>, ) -> Result { - use wb::ColumnValue::*; - Ok(match value.value_type() { - F64Value => { + wb::ColumnValue::F64Value => { let val = value .value_as_f64value() .expect("f64 value should be present") @@ -58,7 +56,7 @@ impl Column { vals.push(Some(val)); Self::F64(vals, StatValues::new(val)) } - I64Value => { + wb::ColumnValue::I64Value => { let val = value .value_as_i64value() .expect("i64 value should be present") @@ -67,7 +65,7 @@ impl Column { vals.push(Some(val)); Self::I64(vals, StatValues::new(val)) } - U64Value => { + wb::ColumnValue::U64Value => { let val = value .value_as_u64value() .expect("u64 value should be present") @@ -76,7 +74,7 @@ impl Column { vals.push(Some(val)); Self::U64(vals, StatValues::new(val)) } - StringValue => { + wb::ColumnValue::StringValue => { let val = value .value_as_string_value() .expect("string value should be present") @@ -86,7 +84,7 @@ impl Column { vals.push(Some(val.to_string())); Self::String(vals, StatValues::new(val.to_string())) } - BoolValue => { + wb::ColumnValue::BoolValue => { let val = value .value_as_bool_value() .expect("bool value should be present") @@ -95,7 +93,7 @@ impl Column { vals.push(Some(val)); Self::Bool(vals, StatValues::new(val)) } - TagValue => { + wb::ColumnValue::TagValue => { let val = value .value_as_tag_value() .expect("tag value should be present") From 52e415ae1b12f1f9132dd26dd5819a676205c6f1 Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Wed, 17 Mar 2021 15:55:32 -0400 Subject: [PATCH 070/104] fix: Verify flatbuffers when restoring from a file, and not after that --- Cargo.lock | 40 ++++++++++++++ internal_types/Cargo.toml | 1 + internal_types/benches/benchmark.rs | 15 +++--- internal_types/src/data.rs | 81 ++++++++++++++++------------- mutable_buffer/src/database.rs | 2 +- mutable_buffer/src/partition.rs | 2 +- mutable_buffer/src/table.rs | 2 +- server/src/buffer.rs | 46 ++++++++++------ 8 files changed, 129 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76c18f99f3..7ee174bb1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,15 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "RustyXML" version = "0.3.0" @@ -1542,6 +1552,7 @@ dependencies = [ "flatbuffers", "generated_types", "influxdb_line_protocol", + "ouroboros", "snafu", "tracing", ] @@ -2141,6 +2152,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ouroboros" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d5c203fe8d786d9d7bec8203cbbff3eb2cf8410c0d70cfd05b3d5f5d545da" +dependencies = [ + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "129943a960e6a08c7e70ca5a09f113c273fe7f10ae8420992c78293e3dffdf65" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "packed_simd_2" version = "0.3.4" @@ -3273,6 +3307,12 @@ dependencies = [ "log", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.15" diff --git a/internal_types/Cargo.toml b/internal_types/Cargo.toml index a0196a31dd..f3b6025e52 100644 --- a/internal_types/Cargo.toml +++ b/internal_types/Cargo.toml @@ -16,6 +16,7 @@ data_types = { path = "../data_types" } flatbuffers = "0.8" generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } +ouroboros = "0.8.3" snafu = "0.6" tracing = "0.1" diff --git a/internal_types/benches/benchmark.rs b/internal_types/benches/benchmark.rs index 7ec4db8a32..abccc96f06 100644 --- a/internal_types/benches/benchmark.rs +++ b/internal_types/benches/benchmark.rs @@ -4,9 +4,12 @@ use data_types::database_rules::{DatabaseRules, PartitionTemplate, TemplatePart} use generated_types::wal as wb; use influxdb_line_protocol::{parse_lines, ParsedLine}; use internal_types::data::{lines_to_replicated_write as lines_to_rw, ReplicatedWrite}; -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt; -use std::time::Duration; +use std::{ + collections::{BTreeMap, BTreeSet}, + convert::TryFrom, + fmt, + time::Duration, +}; const NEXT_ENTRY_NS: i64 = 1_000_000_000; const STARTING_TIMESTAMP_NS: i64 = 0; @@ -61,7 +64,7 @@ fn replicated_write_into_bytes(c: &mut Criterion) { assert_eq!(write.entry_count(), config.partition_count); b.iter(|| { - let _ = write.bytes().len(); + let _ = write.data().len(); }); }, ); @@ -73,7 +76,7 @@ fn bytes_into_struct(c: &mut Criterion) { run_group("bytes_into_struct", c, |lines, rules, config, b| { let write = lines_to_rw(0, 0, &lines, rules); assert_eq!(write.entry_count(), config.partition_count); - let data = write.bytes(); + let data = write.data(); b.iter(|| { let mut db = Db::default(); @@ -160,7 +163,7 @@ struct Db { impl Db { fn deserialize_write(&mut self, data: &[u8]) { - let write = ReplicatedWrite::from(data); + let write = ReplicatedWrite::try_from(data.to_vec()).unwrap(); if let Some(batch) = write.write_buffer_batch() { if let Some(entries) = batch.entries() { diff --git a/internal_types/src/data.rs b/internal_types/src/data.rs index 28a570ae06..56b69111d0 100644 --- a/internal_types/src/data.rs +++ b/internal_types/src/data.rs @@ -6,11 +6,12 @@ use data_types::database_rules::Partitioner; use generated_types::wal as wb; use influxdb_line_protocol::{FieldValue, ParsedLine}; -use std::{collections::BTreeMap, fmt}; +use std::{collections::BTreeMap, convert::TryFrom, fmt}; use chrono::Utc; use crc32fast::Hasher; use flatbuffers::FlatBufferBuilder; +use ouroboros::self_referencing; pub fn type_description(value: wb::ColumnValue) -> &'static str { match value { @@ -26,66 +27,77 @@ pub fn type_description(value: wb::ColumnValue) -> &'static str { } /// A friendlier wrapper to help deal with the Flatbuffers write data -#[derive(Debug, Default, Clone, PartialEq)] +#[self_referencing] +#[derive(Debug, Clone, PartialEq)] pub struct ReplicatedWrite { - pub data: Vec, + data: Vec, + #[borrows(data)] + #[covariant] + fb: wb::ReplicatedWrite<'this>, + #[borrows(data)] + #[covariant] + write_buffer_batch: Option>, } impl ReplicatedWrite { - /// Returns the Flatbuffers struct represented by the raw bytes. - pub fn to_fb(&self) -> wb::ReplicatedWrite<'_> { - flatbuffers::get_root::>(&self.data) - } - /// Returns the Flatbuffers struct for the WriteBufferBatch in the raw bytes /// of the payload of the ReplicatedWrite. - pub fn write_buffer_batch(&self) -> Option> { - match self.to_fb().payload() { - Some(d) => Some(flatbuffers::get_root::>(&d)), - None => None, - } + pub fn write_buffer_batch(&self) -> Option<&wb::WriteBufferBatch<'_>> { + self.borrow_write_buffer_batch().as_ref() + } + + /// Returns the Flatbuffers struct for the ReplicatedWrite + pub fn fb(&self) -> &wb::ReplicatedWrite<'_> { + self.borrow_fb() } /// Returns true if this replicated write matches the writer and sequence. pub fn equal_to_writer_and_sequence(&self, writer_id: u32, sequence_number: u64) -> bool { - let fb = self.to_fb(); - fb.writer() == writer_id && fb.sequence() == sequence_number + self.fb().writer() == writer_id && self.fb().sequence() == sequence_number } /// Returns the writer id and sequence number pub fn writer_and_sequence(&self) -> (u32, u64) { - let fb = self.to_fb(); - (fb.writer(), fb.sequence()) + (self.fb().writer(), self.fb().sequence()) } - /// Returns the serialized bytes for the write. (used for benchmarking) - pub fn bytes(&self) -> &Vec { - &self.data + /// Returns the serialized bytes for the write + pub fn data(&self) -> &[u8] { + self.borrow_data() } /// Returns the number of write buffer entries in this replicated write pub fn entry_count(&self) -> usize { - if let Some(batch) = self.write_buffer_batch() { - if let Some(entries) = batch.entries() { - return entries.len(); - } - } - - 0 + self.write_buffer_batch() + .map_or(0, |wbb| wbb.entries().map_or(0, |entries| entries.len())) } } -impl From<&[u8]> for ReplicatedWrite { - fn from(data: &[u8]) -> Self { - Self { - data: Vec::from(data), +impl TryFrom> for ReplicatedWrite { + type Error = flatbuffers::InvalidFlatbuffer; + + fn try_from(data: Vec) -> Result { + ReplicatedWriteTryBuilder { + data, + fb_builder: |data| flatbuffers::root::>(data), + write_buffer_batch_builder: |data| match flatbuffers::root::>( + data, + )? + .payload() + { + Some(payload) => Ok(Some(flatbuffers::root::>( + &payload, + )?)), + None => Ok(None), + }, } + .try_build() } } impl fmt::Display for ReplicatedWrite { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let fb = self.to_fb(); + let fb = self.fb(); write!( f, "\nwriter:{}, sequence:{}, checksum:{}\n", @@ -192,9 +204,8 @@ pub fn lines_to_replicated_write( fbb.finish(write, None); let (mut data, idx) = fbb.collapse(); - ReplicatedWrite { - data: data.split_off(idx), - } + ReplicatedWrite::try_from(data.split_off(idx)) + .expect("Flatbuffer data just constructed should be valid") } pub fn split_lines_into_write_entry_partitions( diff --git a/mutable_buffer/src/database.rs b/mutable_buffer/src/database.rs index 51fe91e796..ba45dcd57e 100644 --- a/mutable_buffer/src/database.rs +++ b/mutable_buffer/src/database.rs @@ -188,7 +188,7 @@ impl MutableBufferDb { Some(b) => self.write_entries_to_partitions(&b)?, None => { return MissingPayload { - writer: write.to_fb().writer(), + writer: write.fb().writer(), } .fail() } diff --git a/mutable_buffer/src/partition.rs b/mutable_buffer/src/partition.rs index 92797c92ba..ce9f499c52 100644 --- a/mutable_buffer/src/partition.rs +++ b/mutable_buffer/src/partition.rs @@ -929,7 +929,7 @@ mod tests { let lines: Vec<_> = parse_lines(&lp_string).map(|l| l.unwrap()).collect(); let data = split_lines_into_write_entry_partitions(|_| partition.key().into(), &lines); - let batch = flatbuffers::get_root::>(&data); + let batch = flatbuffers::root::>(&data).unwrap(); let entries = batch.entries().unwrap(); for entry in entries { diff --git a/mutable_buffer/src/table.rs b/mutable_buffer/src/table.rs index e10d76d09f..57d0178346 100644 --- a/mutable_buffer/src/table.rs +++ b/mutable_buffer/src/table.rs @@ -808,7 +808,7 @@ mod tests { let data = split_lines_into_write_entry_partitions(chunk_key_func, &lines); - let batch = flatbuffers::get_root::>(&data); + let batch = flatbuffers::root::>(&data).unwrap(); let entries = batch.entries().expect("at least one entry"); for entry in entries { diff --git a/server/src/buffer.rs b/server/src/buffer.rs index 8448130ec2..a2b5e56d78 100644 --- a/server/src/buffer.rs +++ b/server/src/buffer.rs @@ -69,8 +69,16 @@ pub enum Error { #[snafu(display("checksum mismatch for segment"))] ChecksumMismatch, - #[snafu(display("the flatbuffers Segment is invalid"))] - InvalidFlatbuffersSegment, + #[snafu(display("the flatbuffers Segment is invalid: {}", source))] + InvalidFlatbuffersSegment { + source: flatbuffers::InvalidFlatbuffer, + }, + + #[snafu(display("the flatbuffers size is too small; only found {} bytes", bytes))] + FlatbuffersSegmentTooSmall { bytes: usize }, + + #[snafu(display("the flatbuffers Segment is missing an expected value for {}", field))] + FlatbuffersMissingField { field: String }, } pub type Result = std::result::Result; @@ -111,7 +119,7 @@ impl Buffer { /// by accepting the write, the oldest (first) of the closed segments /// will be dropped, if it is persisted. Otherwise, an error is returned. pub fn append(&mut self, write: Arc) -> Result>> { - let write_size = u64::try_from(write.data.len()) + let write_size = u64::try_from(write.data().len()) .expect("appended data must be less than a u64 in length"); while self.current_size + write_size > self.max_size { @@ -310,7 +318,7 @@ impl Segment { let (writer_id, sequence_number) = write.writer_and_sequence(); self.validate_and_update_sequence_summary(writer_id, sequence_number)?; - let size = write.data.len(); + let size = write.data().len(); let size = u64::try_from(size).expect("appended data must be less than a u64 in length"); self.size += size; @@ -418,7 +426,7 @@ impl Segment { .writes .iter() .map(|rw| { - let payload = fbb.create_vector_direct(&rw.data); + let payload = fbb.create_vector_direct(rw.data()); wal::ReplicatedWriteData::create( &mut fbb, &wal::ReplicatedWriteDataArgs { @@ -483,7 +491,7 @@ impl Segment { /// deserializes it into a Segment struct. pub fn from_file_bytes(data: &[u8]) -> Result { if data.len() < std::mem::size_of::() { - return Err(Error::InvalidFlatbuffersSegment); + return FlatbuffersSegmentTooSmall { bytes: data.len() }.fail(); } let (data, checksum) = data.split_at(data.len() - std::mem::size_of::()); @@ -501,15 +509,21 @@ impl Segment { .decompress_vec(data) .context(UnableToDecompressData)?; - let fb_segment = flatbuffers::get_root::>(&data); + // Use verified flatbuffer functionality here + let fb_segment = + flatbuffers::root::>(&data).context(InvalidFlatbuffersSegment)?; - let writes = fb_segment.writes().context(InvalidFlatbuffersSegment)?; + let writes = fb_segment + .writes() + .context(FlatbuffersMissingField { field: "writes" })?; let mut segment = Self::new_with_capacity(fb_segment.id(), writes.len()); for w in writes { - let data = w.payload().context(InvalidFlatbuffersSegment)?; - let rw = ReplicatedWrite { - data: data.to_vec(), - }; + let data = w + .payload() + .context(FlatbuffersMissingField { field: "payload" })? + .to_vec(); + let rw = ReplicatedWrite::try_from(data).context(InvalidFlatbuffersSegment)?; + segment.append(Arc::new(rw))?; } @@ -579,7 +593,7 @@ mod tests { let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let write = lp_to_replicated_write(1, 1, "cpu val=1 10"); - let size = write.data.len() as u64; + let size = write.data().len() as u64; assert_eq!(0, buf.size()); let segment = buf.append(write).unwrap(); assert_eq!(size, buf.size()); @@ -732,7 +746,7 @@ mod tests { fn all_writes_since() { let max = 1 << 63; let write = lp_to_replicated_write(1, 1, "cpu val=1 10"); - let segment = (write.data.len() + 1) as u64; + let segment = (write.data().len() + 1) as u64; let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let segment = buf.append(write).unwrap(); @@ -789,7 +803,7 @@ mod tests { fn writes_since() { let max = 1 << 63; let write = lp_to_replicated_write(1, 1, "cpu val=1 10"); - let segment = (write.data.len() + 1) as u64; + let segment = (write.data().len() + 1) as u64; let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let segment = buf.append(write).unwrap(); @@ -836,7 +850,7 @@ mod tests { fn returns_error_if_sequence_decreases() { let max = 1 << 63; let write = lp_to_replicated_write(1, 3, "cpu val=1 10"); - let segment = (write.data.len() + 1) as u64; + let segment = (write.data().len() + 1) as u64; let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let segment = buf.append(write).unwrap(); From e463d7b8e20ab69a0553eea266d7d8bb3f18354a Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Wed, 17 Mar 2021 16:11:22 -0400 Subject: [PATCH 071/104] fix: Update flatc used in the CI Docker image --- docker/Dockerfile.ci | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index f4f7cea9b7..97987336a2 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -10,15 +10,16 @@ ## # Build any binaries that can be copied into the CI image -# Note we build flatbuffers from source (pinned to a particualar version) +# Note we build flatbuffers from source (pinned to a particualar SHA) FROM rust:slim-buster AS flatc -ARG flatbuffers_version="v1.12.0" +ARG flatbuffers_version="86401e00" RUN apt-get update \ && mkdir -p /usr/share/man/man1 \ && apt-get install -y \ git make clang cmake llvm \ --no-install-recommends \ - && git clone -b ${flatbuffers_version} -- https://github.com/google/flatbuffers.git /usr/local/src/flatbuffers \ + && git clone -b master -- https://github.com/google/flatbuffers.git /usr/local/src/flatbuffers \ + && git -C /usr/local/src/flatbuffers reset --hard ${flatbuffers_version} \ && cmake -S /usr/local/src/flatbuffers -B /usr/local/src/flatbuffers \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ From c47a252c150fe348adf013c9eb7b03de862d38ad Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Thu, 18 Mar 2021 13:43:58 -0400 Subject: [PATCH 072/104] feat: Check in generated WAL flatbuffers code --- README.md | 30 - docs/README.md | 2 + docs/regenerating_flatbuffers.md | 7 + generated_types/.gitignore | 2 + generated_types/Cargo.toml | 2 + generated_types/build.rs | 36 +- generated_types/regenerate-flatbuffers.sh | 49 + generated_types/src/lib.rs | 4 +- generated_types/src/wal_generated.rs | 3220 +++++++++++++++++++++ internal_types/Cargo.toml | 4 +- mutable_buffer/Cargo.toml | 2 + server/Cargo.toml | 2 + 12 files changed, 3293 insertions(+), 67 deletions(-) create mode 100644 docs/regenerating_flatbuffers.md create mode 100644 generated_types/.gitignore create mode 100755 generated_types/regenerate-flatbuffers.sh create mode 100644 generated_types/src/wal_generated.rs diff --git a/README.md b/README.md index e33706c6b2..38f24d12be 100644 --- a/README.md +++ b/README.md @@ -80,36 +80,6 @@ rustc --version and you should see a nightly version of Rust! -### Installing `flatc` - -InfluxDB IOx uses the [FlatBuffer] serialization format for its write-ahead log. The [`flatc` -compiler] reads the schema in `generated_types/wal.fbs` and generates the corresponding Rust code. - -As of this writing (2020-03-15), we are using the `flatbuffers` crate at version 0.8.3 to match -Arrow and reduce our build times. This crate requires `flatc` to be built from source from at least -[86401e0], the same commit that changed the crate verison to 0.8.3. - -You can build `flatc` [using `cmake` as recommended in the building guide][flatc-cmake] or [using -`bazel` as Arrow does][flatc-bazel]. Make sure the resulting executable is in your `PATH`. - -You should be able to run: - -```shell -flatc --version -``` - -and see version 1.12.0 displayed (even though we've built from source at a commit newer than that -released version). - -You won't have to run `flatc` directly; once it's available, Rust's Cargo build tool manages the -compilation process by calling `flatc` for you. - -[FlatBuffer]: https://google.github.io/flatbuffers/ -[`flatc` compiler]: https://google.github.io/flatbuffers/flatbuffers_guide_using_schema_compiler.html -[86401e0]: https://github.com/google/flatbuffers/commit/86401e078d0746d2381735415f8c2dfe849f3f52#diff-c25804a2cc9060430107b2e04f4ec904ed08b798b1bb3f8db963724f91cea522R3 -[flatc-cmake]: https://google.github.io/flatbuffers/flatbuffers_guide_building.html -[flatc-bazel]: https://github.com/apache/arrow/blob/9a80565dbb8d8abe46c1477d684eabb9ce1bb0ff/rust/arrow/regen.sh#L24-L53 - ### Installing `clang` An installation of `clang` is required to build the [`croaring`] dependency - if diff --git a/docs/README.md b/docs/README.md index c6608339df..c14e8f3e52 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,3 +22,5 @@ We hold monthly Tech Talks that explain the project's technical underpinnings. Y * Thoughts on using multiple cores: [multi_core_tasks.md](multi_core_tasks.md) * [Query Engine Docs](../query/README.md) * [Testing documentation](testing.md) for developers of IOx +* [Regenerating Flatbuffers code](regenerating_flatbuffers.md) when updating the version of the `flatbuffers` crate + diff --git a/docs/regenerating_flatbuffers.md b/docs/regenerating_flatbuffers.md new file mode 100644 index 0000000000..1c5f8a07f5 --- /dev/null +++ b/docs/regenerating_flatbuffers.md @@ -0,0 +1,7 @@ +# Regenerating Flatbuffers code + +When updating the version of the [flatbuffers](https://crates.io/crates/flatbuffers) Rust crate used as a dependency in the IOx workspace, the generated Rust code in `generated_types/src/wal_generated.rs` also needs to be updated in sync. + +To update the generated code, edit `generated_types/regenerate-flatbuffers.sh` and set the `FB_COMMIT` variable at the top of the file to the commit SHA of the same commit in the [flatbuffers repository](https://github.com/google/flatbuffers) where the `flatbuffers` Rust crate version was updated. This ensures we'll be [using the same version of `flatc` that the crate was tested with](https://github.com/google/flatbuffers/issues/6199#issuecomment-714562121). + +Then run the `generated_types/regenerate-flatbuffers.sh` script and check in any changes. Check the whole project builds. diff --git a/generated_types/.gitignore b/generated_types/.gitignore new file mode 100644 index 0000000000..3954622303 --- /dev/null +++ b/generated_types/.gitignore @@ -0,0 +1,2 @@ +.flatbuffers + diff --git a/generated_types/Cargo.toml b/generated_types/Cargo.toml index d99e03ff37..af4a631039 100644 --- a/generated_types/Cargo.toml +++ b/generated_types/Cargo.toml @@ -5,6 +5,8 @@ authors = ["Paul Dix "] edition = "2018" [dependencies] # In alphabetical order +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate flatbuffers = "0.8" futures = "0.3.1" prost = "0.7" diff --git a/generated_types/build.rs b/generated_types/build.rs index 202f9c693f..38e65d59e0 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -1,10 +1,6 @@ -//! Compiles Protocol Buffers and FlatBuffers schema definitions into -//! native Rust types. +//! Compiles Protocol Buffers into native Rust types. -use std::{ - path::{Path, PathBuf}, - process::Command, -}; +use std::path::{Path, PathBuf}; type Error = Box; type Result = std::result::Result; @@ -13,7 +9,6 @@ fn main() -> Result<()> { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("protos"); generate_grpc_types(&root)?; - generate_wal_types(&root)?; Ok(()) } @@ -66,30 +61,3 @@ fn generate_grpc_types(root: &Path) -> Result<()> { Ok(()) } - -/// Schema used in the WAL -/// -/// Creates `wal_generated.rs` -fn generate_wal_types(root: &Path) -> Result<()> { - let wal_file = root.join("wal.fbs"); - - println!("cargo:rerun-if-changed={}", wal_file.display()); - let out_dir: PathBuf = std::env::var_os("OUT_DIR") - .expect("Could not determine `OUT_DIR`") - .into(); - - let status = Command::new("flatc") - .arg("--rust") - .arg("-o") - .arg(&out_dir) - .arg(wal_file) - .status(); - - match status { - Ok(status) if !status.success() => panic!("`flatc` failed to compile the .fbs to Rust"), - Ok(_status) => {} // Successfully compiled - Err(err) => panic!("Could not execute `flatc`: {}", err), - } - - Ok(()) -} diff --git a/generated_types/regenerate-flatbuffers.sh b/generated_types/regenerate-flatbuffers.sh new file mode 100755 index 0000000000..57fc2b8271 --- /dev/null +++ b/generated_types/regenerate-flatbuffers.sh @@ -0,0 +1,49 @@ +#!/bin/bash -e + +# The commit where the Rust `flatbuffers` crate version was changed to the version in `Cargo.lock` +# Update this, rerun this script, and check in the changes in the generated code when the +# `flatbuffers` crate version is updated. +FB_COMMIT="86401e078d0746d2381735415f8c2dfe849f3f52" + +# Change to the generated_types crate directory, where this script is located +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +pushd $DIR + +echo "Building flatc from source ..." + +FB_URL="https://github.com/google/flatbuffers" +FB_DIR=".flatbuffers" +FLATC="$FB_DIR/bazel-bin/flatc" + +if [ -z $(which bazel) ]; then + echo "bazel is required to build flatc" + exit 1 +fi + +echo "Bazel version: $(bazel version | head -1 | awk -F':' '{print $2}')" + +if [ ! -e $FB_DIR ]; then + echo "git clone $FB_URL ..." + git clone -b master --no-tag $FB_URL $FB_DIR +else + echo "git pull $FB_URL ..." + git -C $FB_DIR pull --ff-only +fi + +echo "hard reset to $FB_COMMIT" +git -C $FB_DIR reset --hard $FB_COMMIT + +pushd $FB_DIR +echo "run: bazel build :flatc ..." +bazel build :flatc +popd + +WAL_FBS="$DIR/protos/wal.fbs" +WAL_RS_DIR="$DIR/src" + +$FLATC --rust -o $WAL_RS_DIR $WAL_FBS + +cargo fmt +popd + +echo "DONE! Please run 'cargo test' and check in any changes." diff --git a/generated_types/src/lib.rs b/generated_types/src/lib.rs index b97ba87bcd..2dafa8de12 100644 --- a/generated_types/src/lib.rs +++ b/generated_types/src/lib.rs @@ -74,7 +74,9 @@ pub mod grpc { } } -include!(concat!(env!("OUT_DIR"), "/wal_generated.rs")); +/// Generated Flatbuffers code for working with the write-ahead log +pub mod wal_generated; +pub use wal_generated::wal; /// gRPC Storage Service pub const STORAGE_SERVICE: &str = "influxdata.platform.storage.Storage"; diff --git a/generated_types/src/wal_generated.rs b/generated_types/src/wal_generated.rs new file mode 100644 index 0000000000..10161fc37e --- /dev/null +++ b/generated_types/src/wal_generated.rs @@ -0,0 +1,3220 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +use std::cmp::Ordering; +use std::mem; + +extern crate flatbuffers; +use self::flatbuffers::EndianScalar; + +#[allow(unused_imports, dead_code)] +pub mod wal { + + use std::cmp::Ordering; + use std::mem; + + extern crate flatbuffers; + use self::flatbuffers::EndianScalar; + + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MIN_POINT_VALUE: u8 = 0; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MAX_POINT_VALUE: u8 = 5; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + #[allow(non_camel_case_types)] + pub const ENUM_VALUES_POINT_VALUE: [PointValue; 6] = [ + PointValue::NONE, + PointValue::I64Value, + PointValue::U64Value, + PointValue::F64Value, + PointValue::BoolValue, + PointValue::StringValue, + ]; + + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] + #[repr(transparent)] + pub struct PointValue(pub u8); + #[allow(non_upper_case_globals)] + impl PointValue { + pub const NONE: Self = Self(0); + pub const I64Value: Self = Self(1); + pub const U64Value: Self = Self(2); + pub const F64Value: Self = Self(3); + pub const BoolValue: Self = Self(4); + pub const StringValue: Self = Self(5); + + pub const ENUM_MIN: u8 = 0; + pub const ENUM_MAX: u8 = 5; + pub const ENUM_VALUES: &'static [Self] = &[ + Self::NONE, + Self::I64Value, + Self::U64Value, + Self::F64Value, + Self::BoolValue, + Self::StringValue, + ]; + /// Returns the variant's name or "" if unknown. + pub fn variant_name(self) -> Option<&'static str> { + match self { + Self::NONE => Some("NONE"), + Self::I64Value => Some("I64Value"), + Self::U64Value => Some("U64Value"), + Self::F64Value => Some("F64Value"), + Self::BoolValue => Some("BoolValue"), + Self::StringValue => Some("StringValue"), + _ => None, + } + } + } + impl std::fmt::Debug for PointValue { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(name) = self.variant_name() { + f.write_str(name) + } else { + f.write_fmt(format_args!("", self.0)) + } + } + } + impl<'a> flatbuffers::Follow<'a> for PointValue { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + let b = flatbuffers::read_scalar_at::(buf, loc); + Self(b) + } + } + + impl flatbuffers::Push for PointValue { + type Output = PointValue; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, self.0); + } + } + + impl flatbuffers::EndianScalar for PointValue { + #[inline] + fn to_little_endian(self) -> Self { + let b = u8::to_le(self.0); + Self(b) + } + #[inline] + fn from_little_endian(self) -> Self { + let b = u8::from_le(self.0); + Self(b) + } + } + + impl<'a> flatbuffers::Verifiable for PointValue { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + u8::run_verifier(v, pos) + } + } + + impl flatbuffers::SimpleToVerifyInSlice for PointValue {} + pub struct PointValueUnionTableOffset {} + + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MIN_COLUMN_TYPE: i8 = 0; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MAX_COLUMN_TYPE: i8 = 5; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + #[allow(non_camel_case_types)] + pub const ENUM_VALUES_COLUMN_TYPE: [ColumnType; 6] = [ + ColumnType::I64, + ColumnType::U64, + ColumnType::F64, + ColumnType::Tag, + ColumnType::String, + ColumnType::Bool, + ]; + + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] + #[repr(transparent)] + pub struct ColumnType(pub i8); + #[allow(non_upper_case_globals)] + impl ColumnType { + pub const I64: Self = Self(0); + pub const U64: Self = Self(1); + pub const F64: Self = Self(2); + pub const Tag: Self = Self(3); + pub const String: Self = Self(4); + pub const Bool: Self = Self(5); + + pub const ENUM_MIN: i8 = 0; + pub const ENUM_MAX: i8 = 5; + pub const ENUM_VALUES: &'static [Self] = &[ + Self::I64, + Self::U64, + Self::F64, + Self::Tag, + Self::String, + Self::Bool, + ]; + /// Returns the variant's name or "" if unknown. + pub fn variant_name(self) -> Option<&'static str> { + match self { + Self::I64 => Some("I64"), + Self::U64 => Some("U64"), + Self::F64 => Some("F64"), + Self::Tag => Some("Tag"), + Self::String => Some("String"), + Self::Bool => Some("Bool"), + _ => None, + } + } + } + impl std::fmt::Debug for ColumnType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(name) = self.variant_name() { + f.write_str(name) + } else { + f.write_fmt(format_args!("", self.0)) + } + } + } + impl<'a> flatbuffers::Follow<'a> for ColumnType { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + let b = flatbuffers::read_scalar_at::(buf, loc); + Self(b) + } + } + + impl flatbuffers::Push for ColumnType { + type Output = ColumnType; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, self.0); + } + } + + impl flatbuffers::EndianScalar for ColumnType { + #[inline] + fn to_little_endian(self) -> Self { + let b = i8::to_le(self.0); + Self(b) + } + #[inline] + fn from_little_endian(self) -> Self { + let b = i8::from_le(self.0); + Self(b) + } + } + + impl<'a> flatbuffers::Verifiable for ColumnType { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + i8::run_verifier(v, pos) + } + } + + impl flatbuffers::SimpleToVerifyInSlice for ColumnType {} + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MIN_COLUMN_VALUE: u8 = 0; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MAX_COLUMN_VALUE: u8 = 6; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + #[allow(non_camel_case_types)] + pub const ENUM_VALUES_COLUMN_VALUE: [ColumnValue; 7] = [ + ColumnValue::NONE, + ColumnValue::TagValue, + ColumnValue::I64Value, + ColumnValue::U64Value, + ColumnValue::F64Value, + ColumnValue::BoolValue, + ColumnValue::StringValue, + ]; + + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] + #[repr(transparent)] + pub struct ColumnValue(pub u8); + #[allow(non_upper_case_globals)] + impl ColumnValue { + pub const NONE: Self = Self(0); + pub const TagValue: Self = Self(1); + pub const I64Value: Self = Self(2); + pub const U64Value: Self = Self(3); + pub const F64Value: Self = Self(4); + pub const BoolValue: Self = Self(5); + pub const StringValue: Self = Self(6); + + pub const ENUM_MIN: u8 = 0; + pub const ENUM_MAX: u8 = 6; + pub const ENUM_VALUES: &'static [Self] = &[ + Self::NONE, + Self::TagValue, + Self::I64Value, + Self::U64Value, + Self::F64Value, + Self::BoolValue, + Self::StringValue, + ]; + /// Returns the variant's name or "" if unknown. + pub fn variant_name(self) -> Option<&'static str> { + match self { + Self::NONE => Some("NONE"), + Self::TagValue => Some("TagValue"), + Self::I64Value => Some("I64Value"), + Self::U64Value => Some("U64Value"), + Self::F64Value => Some("F64Value"), + Self::BoolValue => Some("BoolValue"), + Self::StringValue => Some("StringValue"), + _ => None, + } + } + } + impl std::fmt::Debug for ColumnValue { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(name) = self.variant_name() { + f.write_str(name) + } else { + f.write_fmt(format_args!("", self.0)) + } + } + } + impl<'a> flatbuffers::Follow<'a> for ColumnValue { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + let b = flatbuffers::read_scalar_at::(buf, loc); + Self(b) + } + } + + impl flatbuffers::Push for ColumnValue { + type Output = ColumnValue; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, self.0); + } + } + + impl flatbuffers::EndianScalar for ColumnValue { + #[inline] + fn to_little_endian(self) -> Self { + let b = u8::to_le(self.0); + Self(b) + } + #[inline] + fn from_little_endian(self) -> Self { + let b = u8::from_le(self.0); + Self(b) + } + } + + impl<'a> flatbuffers::Verifiable for ColumnValue { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + u8::run_verifier(v, pos) + } + } + + impl flatbuffers::SimpleToVerifyInSlice for ColumnValue {} + pub struct ColumnValueUnionTableOffset {} + + pub enum EntryOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Entry<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Entry<'a> { + type Inner = Entry<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Entry<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Entry { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args EntryArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = EntryBuilder::new(_fbb); + if let Some(x) = args.entry_type { + builder.add_entry_type(x); + } + builder.finish() + } + + pub const VT_ENTRY_TYPE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn entry_type(&self) -> Option> { + self._tab + .get::>(Entry::VT_ENTRY_TYPE, None) + } + } + + impl flatbuffers::Verifiable for Entry<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"entry_type", + Self::VT_ENTRY_TYPE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct EntryArgs<'a> { + pub entry_type: Option>>, + } + impl<'a> Default for EntryArgs<'a> { + #[inline] + fn default() -> Self { + EntryArgs { entry_type: None } + } + } + pub struct EntryBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> EntryBuilder<'a, 'b> { + #[inline] + pub fn add_entry_type(&mut self, entry_type: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>( + Entry::VT_ENTRY_TYPE, + entry_type, + ); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> EntryBuilder<'a, 'b> { + let start = _fbb.start_table(); + EntryBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Entry<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Entry"); + ds.field("entry_type", &self.entry_type()); + ds.finish() + } + } + pub enum EntryTypeOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct EntryType<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for EntryType<'a> { + type Inner = EntryType<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> EntryType<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + EntryType { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args EntryTypeArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = EntryTypeBuilder::new(_fbb); + if let Some(x) = args.delete { + builder.add_delete(x); + } + if let Some(x) = args.write { + builder.add_write(x); + } + builder.finish() + } + + pub const VT_WRITE: flatbuffers::VOffsetT = 4; + pub const VT_DELETE: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn write(&self) -> Option> { + self._tab + .get::>(EntryType::VT_WRITE, None) + } + #[inline] + pub fn delete(&self) -> Option> { + self._tab + .get::>(EntryType::VT_DELETE, None) + } + } + + impl flatbuffers::Verifiable for EntryType<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"write", + Self::VT_WRITE, + false, + )? + .visit_field::>( + &"delete", + Self::VT_DELETE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct EntryTypeArgs<'a> { + pub write: Option>>, + pub delete: Option>>, + } + impl<'a> Default for EntryTypeArgs<'a> { + #[inline] + fn default() -> Self { + EntryTypeArgs { + write: None, + delete: None, + } + } + } + pub struct EntryTypeBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> EntryTypeBuilder<'a, 'b> { + #[inline] + pub fn add_write(&mut self, write: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(EntryType::VT_WRITE, write); + } + #[inline] + pub fn add_delete(&mut self, delete: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(EntryType::VT_DELETE, delete); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> EntryTypeBuilder<'a, 'b> { + let start = _fbb.start_table(); + EntryTypeBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for EntryType<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("EntryType"); + ds.field("write", &self.write()); + ds.field("delete", &self.delete()); + ds.finish() + } + } + pub enum WriteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Write<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Write<'a> { + type Inner = Write<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Write<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Write { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBuilder::new(_fbb); + if let Some(x) = args.points { + builder.add_points(x); + } + builder.finish() + } + + pub const VT_POINTS: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn points( + &self, + ) -> Option>>> { + self._tab.get::>, + >>(Write::VT_POINTS, None) + } + } + + impl flatbuffers::Verifiable for Write<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>, + >>(&"points", Self::VT_POINTS, false)? + .finish(); + Ok(()) + } + } + pub struct WriteArgs<'a> { + pub points: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for WriteArgs<'a> { + #[inline] + fn default() -> Self { + WriteArgs { points: None } + } + } + pub struct WriteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBuilder<'a, 'b> { + #[inline] + pub fn add_points( + &mut self, + points: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(Write::VT_POINTS, points); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> WriteBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Write<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Write"); + ds.field("points", &self.points()); + ds.finish() + } + } + pub enum I64ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct I64Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for I64Value<'a> { + type Inner = I64Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> I64Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + I64Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args I64ValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = I64ValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> i64 { + self._tab.get::(I64Value::VT_VALUE, Some(0)).unwrap() + } + } + + impl flatbuffers::Verifiable for I64Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct I64ValueArgs { + pub value: i64, + } + impl<'a> Default for I64ValueArgs { + #[inline] + fn default() -> Self { + I64ValueArgs { value: 0 } + } + } + pub struct I64ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> I64ValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: i64) { + self.fbb_.push_slot::(I64Value::VT_VALUE, value, 0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> I64ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + I64ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for I64Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("I64Value"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum U64ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct U64Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for U64Value<'a> { + type Inner = U64Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> U64Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + U64Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args U64ValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = U64ValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> u64 { + self._tab.get::(U64Value::VT_VALUE, Some(0)).unwrap() + } + } + + impl flatbuffers::Verifiable for U64Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct U64ValueArgs { + pub value: u64, + } + impl<'a> Default for U64ValueArgs { + #[inline] + fn default() -> Self { + U64ValueArgs { value: 0 } + } + } + pub struct U64ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> U64ValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: u64) { + self.fbb_.push_slot::(U64Value::VT_VALUE, value, 0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> U64ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + U64ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for U64Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("U64Value"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum F64ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct F64Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for F64Value<'a> { + type Inner = F64Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> F64Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + F64Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args F64ValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = F64ValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> f64 { + self._tab.get::(F64Value::VT_VALUE, Some(0.0)).unwrap() + } + } + + impl flatbuffers::Verifiable for F64Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct F64ValueArgs { + pub value: f64, + } + impl<'a> Default for F64ValueArgs { + #[inline] + fn default() -> Self { + F64ValueArgs { value: 0.0 } + } + } + pub struct F64ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> F64ValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: f64) { + self.fbb_.push_slot::(F64Value::VT_VALUE, value, 0.0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> F64ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + F64ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for F64Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("F64Value"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum BoolValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct BoolValue<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for BoolValue<'a> { + type Inner = BoolValue<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> BoolValue<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + BoolValue { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args BoolValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = BoolValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> bool { + self._tab + .get::(BoolValue::VT_VALUE, Some(false)) + .unwrap() + } + } + + impl flatbuffers::Verifiable for BoolValue<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct BoolValueArgs { + pub value: bool, + } + impl<'a> Default for BoolValueArgs { + #[inline] + fn default() -> Self { + BoolValueArgs { value: false } + } + } + pub struct BoolValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> BoolValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: bool) { + self.fbb_ + .push_slot::(BoolValue::VT_VALUE, value, false); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BoolValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + BoolValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for BoolValue<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("BoolValue"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum StringValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct StringValue<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for StringValue<'a> { + type Inner = StringValue<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> StringValue<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + StringValue { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args StringValueArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = StringValueBuilder::new(_fbb); + if let Some(x) = args.value { + builder.add_value(x); + } + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> Option<&'a str> { + self._tab + .get::>(StringValue::VT_VALUE, None) + } + } + + impl flatbuffers::Verifiable for StringValue<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct StringValueArgs<'a> { + pub value: Option>, + } + impl<'a> Default for StringValueArgs<'a> { + #[inline] + fn default() -> Self { + StringValueArgs { value: None } + } + } + pub struct StringValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> StringValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(StringValue::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> StringValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + StringValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for StringValue<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("StringValue"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum PointOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Point<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Point<'a> { + type Inner = Point<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Point<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Point { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args PointArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = PointBuilder::new(_fbb); + builder.add_time(args.time); + if let Some(x) = args.value { + builder.add_value(x); + } + if let Some(x) = args.key { + builder.add_key(x); + } + builder.add_value_type(args.value_type); + builder.finish() + } + + pub const VT_KEY: flatbuffers::VOffsetT = 4; + pub const VT_TIME: flatbuffers::VOffsetT = 6; + pub const VT_VALUE_TYPE: flatbuffers::VOffsetT = 8; + pub const VT_VALUE: flatbuffers::VOffsetT = 10; + + #[inline] + pub fn key(&self) -> Option<&'a str> { + self._tab + .get::>(Point::VT_KEY, None) + } + #[inline] + pub fn time(&self) -> i64 { + self._tab.get::(Point::VT_TIME, Some(0)).unwrap() + } + #[inline] + pub fn value_type(&self) -> PointValue { + self._tab + .get::(Point::VT_VALUE_TYPE, Some(PointValue::NONE)) + .unwrap() + } + #[inline] + pub fn value(&self) -> Option> { + self._tab + .get::>>(Point::VT_VALUE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn value_as_i64value(&self) -> Option> { + if self.value_type() == PointValue::I64Value { + self.value().map(I64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_u64value(&self) -> Option> { + if self.value_type() == PointValue::U64Value { + self.value().map(U64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_f64value(&self) -> Option> { + if self.value_type() == PointValue::F64Value { + self.value().map(F64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_bool_value(&self) -> Option> { + if self.value_type() == PointValue::BoolValue { + self.value().map(BoolValue::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_string_value(&self) -> Option> { + if self.value_type() == PointValue::StringValue { + self.value().map(StringValue::init_from_table) + } else { + None + } + } + } + + impl flatbuffers::Verifiable for Point<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"key", Self::VT_KEY, false)? + .visit_field::(&"time", Self::VT_TIME, false)? + .visit_union::( + &"value_type", + Self::VT_VALUE_TYPE, + &"value", + Self::VT_VALUE, + false, + |key, v, pos| match key { + PointValue::I64Value => v + .verify_union_variant::>( + "PointValue::I64Value", + pos, + ), + PointValue::U64Value => v + .verify_union_variant::>( + "PointValue::U64Value", + pos, + ), + PointValue::F64Value => v + .verify_union_variant::>( + "PointValue::F64Value", + pos, + ), + PointValue::BoolValue => v + .verify_union_variant::>( + "PointValue::BoolValue", + pos, + ), + PointValue::StringValue => v + .verify_union_variant::>( + "PointValue::StringValue", + pos, + ), + _ => Ok(()), + }, + )? + .finish(); + Ok(()) + } + } + pub struct PointArgs<'a> { + pub key: Option>, + pub time: i64, + pub value_type: PointValue, + pub value: Option>, + } + impl<'a> Default for PointArgs<'a> { + #[inline] + fn default() -> Self { + PointArgs { + key: None, + time: 0, + value_type: PointValue::NONE, + value: None, + } + } + } + pub struct PointBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> PointBuilder<'a, 'b> { + #[inline] + pub fn add_key(&mut self, key: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(Point::VT_KEY, key); + } + #[inline] + pub fn add_time(&mut self, time: i64) { + self.fbb_.push_slot::(Point::VT_TIME, time, 0); + } + #[inline] + pub fn add_value_type(&mut self, value_type: PointValue) { + self.fbb_ + .push_slot::(Point::VT_VALUE_TYPE, value_type, PointValue::NONE); + } + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset) { + self.fbb_ + .push_slot_always::>(Point::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PointBuilder<'a, 'b> { + let start = _fbb.start_table(); + PointBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Point<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Point"); + ds.field("key", &self.key()); + ds.field("time", &self.time()); + ds.field("value_type", &self.value_type()); + match self.value_type() { + PointValue::I64Value => { + if let Some(x) = self.value_as_i64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::U64Value => { + if let Some(x) = self.value_as_u64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::F64Value => { + if let Some(x) = self.value_as_f64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::BoolValue => { + if let Some(x) = self.value_as_bool_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::StringValue => { + if let Some(x) = self.value_as_string_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + _ => { + let x: Option<()> = None; + ds.field("value", &x) + } + }; + ds.finish() + } + } + pub enum DeleteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Delete<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Delete<'a> { + type Inner = Delete<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Delete<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Delete { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args DeleteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = DeleteBuilder::new(_fbb); + builder.add_stop_time(args.stop_time); + builder.add_start_time(args.start_time); + if let Some(x) = args.predicate { + builder.add_predicate(x); + } + builder.finish() + } + + pub const VT_PREDICATE: flatbuffers::VOffsetT = 4; + pub const VT_START_TIME: flatbuffers::VOffsetT = 6; + pub const VT_STOP_TIME: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn predicate(&self) -> Option<&'a str> { + self._tab + .get::>(Delete::VT_PREDICATE, None) + } + #[inline] + pub fn start_time(&self) -> i64 { + self._tab + .get::(Delete::VT_START_TIME, Some(0)) + .unwrap() + } + #[inline] + pub fn stop_time(&self) -> i64 { + self._tab.get::(Delete::VT_STOP_TIME, Some(0)).unwrap() + } + } + + impl flatbuffers::Verifiable for Delete<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"predicate", + Self::VT_PREDICATE, + false, + )? + .visit_field::(&"start_time", Self::VT_START_TIME, false)? + .visit_field::(&"stop_time", Self::VT_STOP_TIME, false)? + .finish(); + Ok(()) + } + } + pub struct DeleteArgs<'a> { + pub predicate: Option>, + pub start_time: i64, + pub stop_time: i64, + } + impl<'a> Default for DeleteArgs<'a> { + #[inline] + fn default() -> Self { + DeleteArgs { + predicate: None, + start_time: 0, + stop_time: 0, + } + } + } + pub struct DeleteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> DeleteBuilder<'a, 'b> { + #[inline] + pub fn add_predicate(&mut self, predicate: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(Delete::VT_PREDICATE, predicate); + } + #[inline] + pub fn add_start_time(&mut self, start_time: i64) { + self.fbb_ + .push_slot::(Delete::VT_START_TIME, start_time, 0); + } + #[inline] + pub fn add_stop_time(&mut self, stop_time: i64) { + self.fbb_ + .push_slot::(Delete::VT_STOP_TIME, stop_time, 0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DeleteBuilder<'a, 'b> { + let start = _fbb.start_table(); + DeleteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Delete<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Delete"); + ds.field("predicate", &self.predicate()); + ds.field("start_time", &self.start_time()); + ds.field("stop_time", &self.stop_time()); + ds.finish() + } + } + pub enum SegmentOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Segment<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Segment<'a> { + type Inner = Segment<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Segment<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Segment { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args SegmentArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = SegmentBuilder::new(_fbb); + builder.add_id(args.id); + if let Some(x) = args.writes { + builder.add_writes(x); + } + builder.add_writer_id(args.writer_id); + builder.finish() + } + + pub const VT_ID: flatbuffers::VOffsetT = 4; + pub const VT_WRITER_ID: flatbuffers::VOffsetT = 6; + pub const VT_WRITES: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn id(&self) -> u64 { + self._tab.get::(Segment::VT_ID, Some(0)).unwrap() + } + #[inline] + pub fn writer_id(&self) -> u32 { + self._tab + .get::(Segment::VT_WRITER_ID, Some(0)) + .unwrap() + } + #[inline] + pub fn writes( + &self, + ) -> Option>>> + { + self._tab.get::>, + >>(Segment::VT_WRITES, None) + } + } + + impl flatbuffers::Verifiable for Segment<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"id", Self::VT_ID, false)? + .visit_field::(&"writer_id", Self::VT_WRITER_ID, false)? + .visit_field::>, + >>(&"writes", Self::VT_WRITES, false)? + .finish(); + Ok(()) + } + } + pub struct SegmentArgs<'a> { + pub id: u64, + pub writer_id: u32, + pub writes: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for SegmentArgs<'a> { + #[inline] + fn default() -> Self { + SegmentArgs { + id: 0, + writer_id: 0, + writes: None, + } + } + } + pub struct SegmentBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> SegmentBuilder<'a, 'b> { + #[inline] + pub fn add_id(&mut self, id: u64) { + self.fbb_.push_slot::(Segment::VT_ID, id, 0); + } + #[inline] + pub fn add_writer_id(&mut self, writer_id: u32) { + self.fbb_ + .push_slot::(Segment::VT_WRITER_ID, writer_id, 0); + } + #[inline] + pub fn add_writes( + &mut self, + writes: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(Segment::VT_WRITES, writes); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> SegmentBuilder<'a, 'b> { + let start = _fbb.start_table(); + SegmentBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Segment<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Segment"); + ds.field("id", &self.id()); + ds.field("writer_id", &self.writer_id()); + ds.field("writes", &self.writes()); + ds.finish() + } + } + pub enum ReplicatedWriteDataOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct ReplicatedWriteData<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for ReplicatedWriteData<'a> { + type Inner = ReplicatedWriteData<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> ReplicatedWriteData<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReplicatedWriteData { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReplicatedWriteDataArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = ReplicatedWriteDataBuilder::new(_fbb); + if let Some(x) = args.payload { + builder.add_payload(x); + } + builder.finish() + } + + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn payload(&self) -> Option<&'a [u8]> { + self._tab + .get::>>( + ReplicatedWriteData::VT_PAYLOAD, + None, + ) + .map(|v| v.safe_slice()) + } + } + + impl flatbuffers::Verifiable for ReplicatedWriteData<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>( + &"payload", + Self::VT_PAYLOAD, + false, + )? + .finish(); + Ok(()) + } + } + pub struct ReplicatedWriteDataArgs<'a> { + pub payload: Option>>, + } + impl<'a> Default for ReplicatedWriteDataArgs<'a> { + #[inline] + fn default() -> Self { + ReplicatedWriteDataArgs { payload: None } + } + } + pub struct ReplicatedWriteDataBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> ReplicatedWriteDataBuilder<'a, 'b> { + #[inline] + pub fn add_payload( + &mut self, + payload: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + ReplicatedWriteData::VT_PAYLOAD, + payload, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> ReplicatedWriteDataBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReplicatedWriteDataBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for ReplicatedWriteData<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("ReplicatedWriteData"); + ds.field("payload", &self.payload()); + ds.finish() + } + } + pub enum WriterSummaryOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriterSummary<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriterSummary<'a> { + type Inner = WriterSummary<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriterSummary<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriterSummary { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriterSummaryArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriterSummaryBuilder::new(_fbb); + builder.add_end_sequence(args.end_sequence); + builder.add_start_sequence(args.start_sequence); + builder.add_writer_id(args.writer_id); + builder.add_missing_sequences(args.missing_sequences); + builder.finish() + } + + pub const VT_WRITER_ID: flatbuffers::VOffsetT = 4; + pub const VT_START_SEQUENCE: flatbuffers::VOffsetT = 6; + pub const VT_END_SEQUENCE: flatbuffers::VOffsetT = 8; + pub const VT_MISSING_SEQUENCES: flatbuffers::VOffsetT = 10; + + #[inline] + pub fn writer_id(&self) -> u64 { + self._tab + .get::(WriterSummary::VT_WRITER_ID, Some(0)) + .unwrap() + } + #[inline] + pub fn start_sequence(&self) -> u64 { + self._tab + .get::(WriterSummary::VT_START_SEQUENCE, Some(0)) + .unwrap() + } + #[inline] + pub fn end_sequence(&self) -> u64 { + self._tab + .get::(WriterSummary::VT_END_SEQUENCE, Some(0)) + .unwrap() + } + #[inline] + pub fn missing_sequences(&self) -> bool { + self._tab + .get::(WriterSummary::VT_MISSING_SEQUENCES, Some(false)) + .unwrap() + } + } + + impl flatbuffers::Verifiable for WriterSummary<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"writer_id", Self::VT_WRITER_ID, false)? + .visit_field::(&"start_sequence", Self::VT_START_SEQUENCE, false)? + .visit_field::(&"end_sequence", Self::VT_END_SEQUENCE, false)? + .visit_field::(&"missing_sequences", Self::VT_MISSING_SEQUENCES, false)? + .finish(); + Ok(()) + } + } + pub struct WriterSummaryArgs { + pub writer_id: u64, + pub start_sequence: u64, + pub end_sequence: u64, + pub missing_sequences: bool, + } + impl<'a> Default for WriterSummaryArgs { + #[inline] + fn default() -> Self { + WriterSummaryArgs { + writer_id: 0, + start_sequence: 0, + end_sequence: 0, + missing_sequences: false, + } + } + } + pub struct WriterSummaryBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriterSummaryBuilder<'a, 'b> { + #[inline] + pub fn add_writer_id(&mut self, writer_id: u64) { + self.fbb_ + .push_slot::(WriterSummary::VT_WRITER_ID, writer_id, 0); + } + #[inline] + pub fn add_start_sequence(&mut self, start_sequence: u64) { + self.fbb_ + .push_slot::(WriterSummary::VT_START_SEQUENCE, start_sequence, 0); + } + #[inline] + pub fn add_end_sequence(&mut self, end_sequence: u64) { + self.fbb_ + .push_slot::(WriterSummary::VT_END_SEQUENCE, end_sequence, 0); + } + #[inline] + pub fn add_missing_sequences(&mut self, missing_sequences: bool) { + self.fbb_.push_slot::( + WriterSummary::VT_MISSING_SEQUENCES, + missing_sequences, + false, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriterSummaryBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriterSummaryBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriterSummary<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriterSummary"); + ds.field("writer_id", &self.writer_id()); + ds.field("start_sequence", &self.start_sequence()); + ds.field("end_sequence", &self.end_sequence()); + ds.field("missing_sequences", &self.missing_sequences()); + ds.finish() + } + } + pub enum ReplicatedWriteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct ReplicatedWrite<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for ReplicatedWrite<'a> { + type Inner = ReplicatedWrite<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> ReplicatedWrite<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReplicatedWrite { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReplicatedWriteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = ReplicatedWriteBuilder::new(_fbb); + builder.add_sequence(args.sequence); + if let Some(x) = args.payload { + builder.add_payload(x); + } + builder.add_checksum(args.checksum); + builder.add_writer(args.writer); + builder.finish() + } + + pub const VT_WRITER: flatbuffers::VOffsetT = 4; + pub const VT_SEQUENCE: flatbuffers::VOffsetT = 6; + pub const VT_CHECKSUM: flatbuffers::VOffsetT = 8; + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 10; + + #[inline] + pub fn writer(&self) -> u32 { + self._tab + .get::(ReplicatedWrite::VT_WRITER, Some(0)) + .unwrap() + } + #[inline] + pub fn sequence(&self) -> u64 { + self._tab + .get::(ReplicatedWrite::VT_SEQUENCE, Some(0)) + .unwrap() + } + #[inline] + pub fn checksum(&self) -> u32 { + self._tab + .get::(ReplicatedWrite::VT_CHECKSUM, Some(0)) + .unwrap() + } + #[inline] + pub fn payload(&self) -> Option<&'a [u8]> { + self._tab + .get::>>( + ReplicatedWrite::VT_PAYLOAD, + None, + ) + .map(|v| v.safe_slice()) + } + } + + impl flatbuffers::Verifiable for ReplicatedWrite<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"writer", Self::VT_WRITER, false)? + .visit_field::(&"sequence", Self::VT_SEQUENCE, false)? + .visit_field::(&"checksum", Self::VT_CHECKSUM, false)? + .visit_field::>>( + &"payload", + Self::VT_PAYLOAD, + false, + )? + .finish(); + Ok(()) + } + } + pub struct ReplicatedWriteArgs<'a> { + pub writer: u32, + pub sequence: u64, + pub checksum: u32, + pub payload: Option>>, + } + impl<'a> Default for ReplicatedWriteArgs<'a> { + #[inline] + fn default() -> Self { + ReplicatedWriteArgs { + writer: 0, + sequence: 0, + checksum: 0, + payload: None, + } + } + } + pub struct ReplicatedWriteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> ReplicatedWriteBuilder<'a, 'b> { + #[inline] + pub fn add_writer(&mut self, writer: u32) { + self.fbb_ + .push_slot::(ReplicatedWrite::VT_WRITER, writer, 0); + } + #[inline] + pub fn add_sequence(&mut self, sequence: u64) { + self.fbb_ + .push_slot::(ReplicatedWrite::VT_SEQUENCE, sequence, 0); + } + #[inline] + pub fn add_checksum(&mut self, checksum: u32) { + self.fbb_ + .push_slot::(ReplicatedWrite::VT_CHECKSUM, checksum, 0); + } + #[inline] + pub fn add_payload( + &mut self, + payload: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + ReplicatedWrite::VT_PAYLOAD, + payload, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> ReplicatedWriteBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReplicatedWriteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for ReplicatedWrite<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("ReplicatedWrite"); + ds.field("writer", &self.writer()); + ds.field("sequence", &self.sequence()); + ds.field("checksum", &self.checksum()); + ds.field("payload", &self.payload()); + ds.finish() + } + } + pub enum WriteBufferBatchOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriteBufferBatch<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriteBufferBatch<'a> { + type Inner = WriteBufferBatch<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriteBufferBatch<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriteBufferBatch { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteBufferBatchArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBufferBatchBuilder::new(_fbb); + if let Some(x) = args.entries { + builder.add_entries(x); + } + builder.finish() + } + + pub const VT_ENTRIES: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn entries( + &self, + ) -> Option>>> + { + self._tab.get::>, + >>(WriteBufferBatch::VT_ENTRIES, None) + } + } + + impl flatbuffers::Verifiable for WriteBufferBatch<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>, + >>(&"entries", Self::VT_ENTRIES, false)? + .finish(); + Ok(()) + } + } + pub struct WriteBufferBatchArgs<'a> { + pub entries: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for WriteBufferBatchArgs<'a> { + #[inline] + fn default() -> Self { + WriteBufferBatchArgs { entries: None } + } + } + pub struct WriteBufferBatchBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBufferBatchBuilder<'a, 'b> { + #[inline] + pub fn add_entries( + &mut self, + entries: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_.push_slot_always::>( + WriteBufferBatch::VT_ENTRIES, + entries, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriteBufferBatchBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBufferBatchBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriteBufferBatch<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriteBufferBatch"); + ds.field("entries", &self.entries()); + ds.finish() + } + } + pub enum WriteBufferEntryOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriteBufferEntry<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriteBufferEntry<'a> { + type Inner = WriteBufferEntry<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriteBufferEntry<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriteBufferEntry { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteBufferEntryArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBufferEntryBuilder::new(_fbb); + if let Some(x) = args.delete { + builder.add_delete(x); + } + if let Some(x) = args.table_batches { + builder.add_table_batches(x); + } + if let Some(x) = args.partition_key { + builder.add_partition_key(x); + } + builder.finish() + } + + pub const VT_PARTITION_KEY: flatbuffers::VOffsetT = 4; + pub const VT_TABLE_BATCHES: flatbuffers::VOffsetT = 6; + pub const VT_DELETE: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn partition_key(&self) -> Option<&'a str> { + self._tab + .get::>(WriteBufferEntry::VT_PARTITION_KEY, None) + } + #[inline] + pub fn table_batches( + &self, + ) -> Option>>> + { + self._tab.get::>, + >>(WriteBufferEntry::VT_TABLE_BATCHES, None) + } + #[inline] + pub fn delete(&self) -> Option> { + self._tab + .get::>( + WriteBufferEntry::VT_DELETE, + None, + ) + } + } + + impl flatbuffers::Verifiable for WriteBufferEntry<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"partition_key", + Self::VT_PARTITION_KEY, + false, + )? + .visit_field::>, + >>(&"table_batches", Self::VT_TABLE_BATCHES, false)? + .visit_field::>( + &"delete", + Self::VT_DELETE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct WriteBufferEntryArgs<'a> { + pub partition_key: Option>, + pub table_batches: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + pub delete: Option>>, + } + impl<'a> Default for WriteBufferEntryArgs<'a> { + #[inline] + fn default() -> Self { + WriteBufferEntryArgs { + partition_key: None, + table_batches: None, + delete: None, + } + } + } + pub struct WriteBufferEntryBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBufferEntryBuilder<'a, 'b> { + #[inline] + pub fn add_partition_key(&mut self, partition_key: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + WriteBufferEntry::VT_PARTITION_KEY, + partition_key, + ); + } + #[inline] + pub fn add_table_batches( + &mut self, + table_batches: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_.push_slot_always::>( + WriteBufferEntry::VT_TABLE_BATCHES, + table_batches, + ); + } + #[inline] + pub fn add_delete(&mut self, delete: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>( + WriteBufferEntry::VT_DELETE, + delete, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriteBufferEntryBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBufferEntryBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriteBufferEntry<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriteBufferEntry"); + ds.field("partition_key", &self.partition_key()); + ds.field("table_batches", &self.table_batches()); + ds.field("delete", &self.delete()); + ds.finish() + } + } + pub enum TableWriteBatchOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct TableWriteBatch<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for TableWriteBatch<'a> { + type Inner = TableWriteBatch<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> TableWriteBatch<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TableWriteBatch { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args TableWriteBatchArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = TableWriteBatchBuilder::new(_fbb); + if let Some(x) = args.rows { + builder.add_rows(x); + } + if let Some(x) = args.name { + builder.add_name(x); + } + builder.finish() + } + + pub const VT_NAME: flatbuffers::VOffsetT = 4; + pub const VT_ROWS: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn name(&self) -> Option<&'a str> { + self._tab + .get::>(TableWriteBatch::VT_NAME, None) + } + #[inline] + pub fn rows( + &self, + ) -> Option>>> { + self._tab.get::>, + >>(TableWriteBatch::VT_ROWS, None) + } + } + + impl flatbuffers::Verifiable for TableWriteBatch<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"name", Self::VT_NAME, false)? + .visit_field::>, + >>(&"rows", Self::VT_ROWS, false)? + .finish(); + Ok(()) + } + } + pub struct TableWriteBatchArgs<'a> { + pub name: Option>, + pub rows: Option< + flatbuffers::WIPOffset>>>, + >, + } + impl<'a> Default for TableWriteBatchArgs<'a> { + #[inline] + fn default() -> Self { + TableWriteBatchArgs { + name: None, + rows: None, + } + } + } + pub struct TableWriteBatchBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> TableWriteBatchBuilder<'a, 'b> { + #[inline] + pub fn add_name(&mut self, name: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(TableWriteBatch::VT_NAME, name); + } + #[inline] + pub fn add_rows( + &mut self, + rows: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(TableWriteBatch::VT_ROWS, rows); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> TableWriteBatchBuilder<'a, 'b> { + let start = _fbb.start_table(); + TableWriteBatchBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for TableWriteBatch<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("TableWriteBatch"); + ds.field("name", &self.name()); + ds.field("rows", &self.rows()); + ds.finish() + } + } + pub enum RowOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Row<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Row<'a> { + type Inner = Row<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Row<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Row { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args RowArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = RowBuilder::new(_fbb); + if let Some(x) = args.values { + builder.add_values(x); + } + builder.finish() + } + + pub const VT_VALUES: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn values( + &self, + ) -> Option>>> { + self._tab.get::>, + >>(Row::VT_VALUES, None) + } + } + + impl flatbuffers::Verifiable for Row<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>, + >>(&"values", Self::VT_VALUES, false)? + .finish(); + Ok(()) + } + } + pub struct RowArgs<'a> { + pub values: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for RowArgs<'a> { + #[inline] + fn default() -> Self { + RowArgs { values: None } + } + } + pub struct RowBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> RowBuilder<'a, 'b> { + #[inline] + pub fn add_values( + &mut self, + values: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(Row::VT_VALUES, values); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> RowBuilder<'a, 'b> { + let start = _fbb.start_table(); + RowBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Row<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Row"); + ds.field("values", &self.values()); + ds.finish() + } + } + pub enum TagValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct TagValue<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for TagValue<'a> { + type Inner = TagValue<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> TagValue<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TagValue { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args TagValueArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = TagValueBuilder::new(_fbb); + if let Some(x) = args.value { + builder.add_value(x); + } + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> Option<&'a str> { + self._tab + .get::>(TagValue::VT_VALUE, None) + } + } + + impl flatbuffers::Verifiable for TagValue<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct TagValueArgs<'a> { + pub value: Option>, + } + impl<'a> Default for TagValueArgs<'a> { + #[inline] + fn default() -> Self { + TagValueArgs { value: None } + } + } + pub struct TagValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> TagValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(TagValue::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> TagValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + TagValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for TagValue<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("TagValue"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Value<'a> { + type Inner = Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ValueArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = ValueBuilder::new(_fbb); + if let Some(x) = args.value { + builder.add_value(x); + } + if let Some(x) = args.column { + builder.add_column(x); + } + builder.add_value_type(args.value_type); + builder.finish() + } + + pub const VT_COLUMN: flatbuffers::VOffsetT = 4; + pub const VT_VALUE_TYPE: flatbuffers::VOffsetT = 6; + pub const VT_VALUE: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn column(&self) -> Option<&'a str> { + self._tab + .get::>(Value::VT_COLUMN, None) + } + #[inline] + pub fn value_type(&self) -> ColumnValue { + self._tab + .get::(Value::VT_VALUE_TYPE, Some(ColumnValue::NONE)) + .unwrap() + } + #[inline] + pub fn value(&self) -> Option> { + self._tab + .get::>>(Value::VT_VALUE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn value_as_tag_value(&self) -> Option> { + if self.value_type() == ColumnValue::TagValue { + self.value().map(TagValue::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_i64value(&self) -> Option> { + if self.value_type() == ColumnValue::I64Value { + self.value().map(I64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_u64value(&self) -> Option> { + if self.value_type() == ColumnValue::U64Value { + self.value().map(U64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_f64value(&self) -> Option> { + if self.value_type() == ColumnValue::F64Value { + self.value().map(F64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_bool_value(&self) -> Option> { + if self.value_type() == ColumnValue::BoolValue { + self.value().map(BoolValue::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_string_value(&self) -> Option> { + if self.value_type() == ColumnValue::StringValue { + self.value().map(StringValue::init_from_table) + } else { + None + } + } + } + + impl flatbuffers::Verifiable for Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"column", + Self::VT_COLUMN, + false, + )? + .visit_union::( + &"value_type", + Self::VT_VALUE_TYPE, + &"value", + Self::VT_VALUE, + false, + |key, v, pos| match key { + ColumnValue::TagValue => v + .verify_union_variant::>( + "ColumnValue::TagValue", + pos, + ), + ColumnValue::I64Value => v + .verify_union_variant::>( + "ColumnValue::I64Value", + pos, + ), + ColumnValue::U64Value => v + .verify_union_variant::>( + "ColumnValue::U64Value", + pos, + ), + ColumnValue::F64Value => v + .verify_union_variant::>( + "ColumnValue::F64Value", + pos, + ), + ColumnValue::BoolValue => v + .verify_union_variant::>( + "ColumnValue::BoolValue", + pos, + ), + ColumnValue::StringValue => v + .verify_union_variant::>( + "ColumnValue::StringValue", + pos, + ), + _ => Ok(()), + }, + )? + .finish(); + Ok(()) + } + } + pub struct ValueArgs<'a> { + pub column: Option>, + pub value_type: ColumnValue, + pub value: Option>, + } + impl<'a> Default for ValueArgs<'a> { + #[inline] + fn default() -> Self { + ValueArgs { + column: None, + value_type: ColumnValue::NONE, + value: None, + } + } + } + pub struct ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> ValueBuilder<'a, 'b> { + #[inline] + pub fn add_column(&mut self, column: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(Value::VT_COLUMN, column); + } + #[inline] + pub fn add_value_type(&mut self, value_type: ColumnValue) { + self.fbb_ + .push_slot::(Value::VT_VALUE_TYPE, value_type, ColumnValue::NONE); + } + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset) { + self.fbb_ + .push_slot_always::>(Value::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Value"); + ds.field("column", &self.column()); + ds.field("value_type", &self.value_type()); + match self.value_type() { + ColumnValue::TagValue => { + if let Some(x) = self.value_as_tag_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::I64Value => { + if let Some(x) = self.value_as_i64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::U64Value => { + if let Some(x) = self.value_as_u64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::F64Value => { + if let Some(x) = self.value_as_f64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::BoolValue => { + if let Some(x) = self.value_as_bool_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::StringValue => { + if let Some(x) = self.value_as_string_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + _ => { + let x: Option<()> = None; + ds.field("value", &x) + } + }; + ds.finish() + } + } + pub enum WriteBufferDeleteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriteBufferDelete<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriteBufferDelete<'a> { + type Inner = WriteBufferDelete<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriteBufferDelete<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriteBufferDelete { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteBufferDeleteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBufferDeleteBuilder::new(_fbb); + if let Some(x) = args.predicate { + builder.add_predicate(x); + } + if let Some(x) = args.table_name { + builder.add_table_name(x); + } + builder.finish() + } + + pub const VT_TABLE_NAME: flatbuffers::VOffsetT = 4; + pub const VT_PREDICATE: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn table_name(&self) -> Option<&'a str> { + self._tab + .get::>(WriteBufferDelete::VT_TABLE_NAME, None) + } + #[inline] + pub fn predicate(&self) -> Option<&'a str> { + self._tab + .get::>(WriteBufferDelete::VT_PREDICATE, None) + } + } + + impl flatbuffers::Verifiable for WriteBufferDelete<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"table_name", + Self::VT_TABLE_NAME, + false, + )? + .visit_field::>( + &"predicate", + Self::VT_PREDICATE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct WriteBufferDeleteArgs<'a> { + pub table_name: Option>, + pub predicate: Option>, + } + impl<'a> Default for WriteBufferDeleteArgs<'a> { + #[inline] + fn default() -> Self { + WriteBufferDeleteArgs { + table_name: None, + predicate: None, + } + } + } + pub struct WriteBufferDeleteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBufferDeleteBuilder<'a, 'b> { + #[inline] + pub fn add_table_name(&mut self, table_name: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + WriteBufferDelete::VT_TABLE_NAME, + table_name, + ); + } + #[inline] + pub fn add_predicate(&mut self, predicate: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + WriteBufferDelete::VT_PREDICATE, + predicate, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriteBufferDeleteBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBufferDeleteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriteBufferDelete<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriteBufferDelete"); + ds.field("table_name", &self.table_name()); + ds.field("predicate", &self.predicate()); + ds.finish() + } + } +} // pub mod wal diff --git a/internal_types/Cargo.toml b/internal_types/Cargo.toml index f3b6025e52..a259be2718 100644 --- a/internal_types/Cargo.toml +++ b/internal_types/Cargo.toml @@ -6,13 +6,13 @@ edition = "2018" description = "InfluxDB IOx internal types, shared between IOx instances" readme = "README.md" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] arrow_deps = { path = "../arrow_deps" } crc32fast = "1.2.0" chrono = { version = "0.4", features = ["serde"] } data_types = { path = "../data_types" } +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate flatbuffers = "0.8" generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } diff --git a/mutable_buffer/Cargo.toml b/mutable_buffer/Cargo.toml index a84fae9357..57c3c0e81b 100644 --- a/mutable_buffer/Cargo.toml +++ b/mutable_buffer/Cargo.toml @@ -18,6 +18,8 @@ arrow_deps = { path = "../arrow_deps" } async-trait = "0.1" chrono = "0.4" data_types = { path = "../data_types" } +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate flatbuffers = "0.8" generated_types = { path = "../generated_types" } internal_types = { path = "../internal_types" } diff --git a/server/Cargo.toml b/server/Cargo.toml index 3e081bf725..7c020bc999 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,6 +11,8 @@ bytes = "1.0" chrono = "0.4" crc32fast = "1.2.0" data_types = { path = "../data_types" } +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate flatbuffers = "0.8" futures = "0.3.7" generated_types = { path = "../generated_types" } From 08727732cc319a108b07dfa8cba29712051b6d92 Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Thu, 18 Mar 2021 21:04:16 -0400 Subject: [PATCH 073/104] fix: Remove more flatc --- README.md | 4 +--- docker/Dockerfile.ci | 17 ----------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/README.md b/README.md index 38f24d12be..0daa900ed7 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,7 @@ We're also hosting monthly tech talks and community office hours on the project ## Quick Start -To compile and run InfluxDB IOx from source, you'll need a Rust compiler and a `flatc` FlatBuffers -compiler. +To compile and run InfluxDB IOx from source, you'll need a Rust compiler and `clang`. ### Build a Docker Image @@ -113,7 +112,6 @@ provided [example](docs/env.example) as a template if you want: cp docs/env.example .env ``` - ### Compiling and Starting the Server InfluxDB IOx is built using Cargo, Rust's package manager and build tool. diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 97987336a2..dcf337d32d 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -9,26 +9,9 @@ # docker build -f docker/Dockerfile.ci . ## -# Build any binaries that can be copied into the CI image -# Note we build flatbuffers from source (pinned to a particualar SHA) -FROM rust:slim-buster AS flatc -ARG flatbuffers_version="86401e00" -RUN apt-get update \ - && mkdir -p /usr/share/man/man1 \ - && apt-get install -y \ - git make clang cmake llvm \ - --no-install-recommends \ - && git clone -b master -- https://github.com/google/flatbuffers.git /usr/local/src/flatbuffers \ - && git -C /usr/local/src/flatbuffers reset --hard ${flatbuffers_version} \ - && cmake -S /usr/local/src/flatbuffers -B /usr/local/src/flatbuffers \ - -G "Unix Makefiles" \ - -DCMAKE_BUILD_TYPE=Release \ - && make -C /usr/local/src/flatbuffers -j $(nproc) flatc - # Build actual image used for CI pipeline FROM rust:slim-buster -COPY --from=flatc /usr/local/src/flatbuffers/flatc /usr/bin/flatc # make Apt non-interactive RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90ci \ && echo 'DPkg::Options "--force-confnew";' >> /etc/apt/apt.conf.d/90ci From 5d497b77f3ee36acc6289c2b26fcf28f54e44c07 Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Thu, 18 Mar 2021 21:05:22 -0400 Subject: [PATCH 074/104] feat: Mark the checked-in flatbuffers code as generated --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 05eca1f076..05de81c3b6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ generated_types/protos/google/ linguist-generated=true generated_types/protos/grpc/ linguist-generated=true +generated_types/src/wal_generated.rs linguist-generated=true From 13437c9d734a3a682be0564843a4696c7141e94e Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Mon, 22 Mar 2021 09:46:39 -0400 Subject: [PATCH 075/104] docs: Reference docs on generated flatbuffers in CONTRIBUTING.md --- CONTRIBUTING.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index daf59984e6..4ee1c6732b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -186,3 +186,20 @@ cargo clippy --all-targets --workspace -- -D warnings [`rustfmt`]: https://github.com/rust-lang/rustfmt [`clippy`]: https://github.com/rust-lang/rust-clippy + +## Upgrading the `flatbuffers` crate + +IOx uses Flatbuffers for its write-ahead log. The structure is defined in +[`generated_types/protos/wal.fbs`]. We have then used the `flatc` Flatbuffers compiler to generate +the corresponding Rust code in [`generated_types/src/wal_generated.rs`], which is checked in to the +repository. + +The checked-in code is compatible with the `flatbuffers` crate version in the `Cargo.lock` file. If +upgrading the version of the `flatbuffers` crate that IOx depends on, the generated code will need +to be updated as well. + +Instructions for updating the generated code are in [`docs/regenerating_flatbuffers.md`]. + +[`generated_types/protos/wal.fbs`]: generated_types/protos/wal.fbs +[`generated_types/src/wal_generated.rs`]: generated_types/src/wal_generated.rs +[`docs/regenerating_flatbuffers.md`]: docs/regenerating_flatbuffers.md From 2242206b2822d3b65b46da69e54f9dedd73b20f9 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 22 Mar 2021 14:44:42 +0100 Subject: [PATCH 076/104] fix(ci): Pass debuginfo=1 to have line numbers in tracebacks --- .github/workflows/rust.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 801f78cff4..5ec1f00efb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,8 +20,9 @@ on: [pull_request] name: ci env: - # Disable debug symbol generation to speed up CI build - RUSTFLAGS: "-C debuginfo=0" + # Disable full debug symbol generation to speed up CI build + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" jobs: From d88e20c7feb00b025a3754d7b3fee3b92818497e Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Fri, 19 Mar 2021 10:34:41 +0000 Subject: [PATCH 077/104] fix: fix bad expected cardinality --- read_buffer/benches/row_group.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/read_buffer/benches/row_group.rs b/read_buffer/benches/row_group.rs index 86dc1d3c00..a19d4915a9 100644 --- a/read_buffer/benches/row_group.rs +++ b/read_buffer/benches/row_group.rs @@ -35,7 +35,7 @@ fn read_group_predicate_all_time(c: &mut Criterion, row_group: &RowGroup, rng: & &time_pred, // grouping columns and expected cardinality vec![ - (vec!["env"], 20), + (vec!["env"], 2), (vec!["env", "data_centre"], 20), (vec!["data_centre", "cluster"], 200), (vec!["cluster", "node_id"], 2000), From 9c48cebf78b96e52bdcaee86432437468816d983 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Thu, 11 Mar 2021 12:41:59 +0000 Subject: [PATCH 078/104] refactor: column-wise aggregates --- read_buffer/src/row_group.rs | 125 +++++++++++++++---------- read_buffer/src/value.rs | 177 ++++++++++++++++++++++++++++++++++- 2 files changed, 254 insertions(+), 48 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index d057eed6c2..d015648f93 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -15,7 +15,7 @@ use crate::column::{cmp::Operator, Column, RowIDs, RowIDsOption}; use crate::schema; use crate::schema::{AggregateType, LogicalDataType, ResultSchema}; use crate::value::{ - AggregateResult, EncodedValues, OwnedValue, Scalar, Value, Values, ValuesIterator, + AggregateResult, AggregateVec, EncodedValues, OwnedValue, Scalar, Value, Values, ValuesIterator, }; use arrow_deps::arrow::record_batch::RecordBatch; use arrow_deps::{ @@ -617,74 +617,101 @@ impl RowGroup { &'a self, dst: &mut ReadAggregateResult<'a>, groupby_encoded_ids: &[Vec], - aggregate_columns_data: &[Values<'a>], + aggregate_input_columns: &[Values<'a>], ) { let total_rows = groupby_encoded_ids[0].len(); assert!(groupby_encoded_ids.iter().all(|x| x.len() == total_rows)); assert!(dst.schema.group_columns.len() <= 4); - // Now begin building the group keys. - let mut groups: HashMap>> = HashMap::default(); + // These vectors will hold the decoded values of each part of each + // group key. They are the output columns of the input columns used for + // the grouping operation. + let mut group_cols_out: Vec>>> = vec![]; + group_cols_out.resize(groupby_encoded_ids.len(), vec![]); - for row in 0..groupby_encoded_ids[0].len() { - // pack each column's encoded value for the row into a packed group - // key. + // Each of these vectors will be used to store each aggregate row-value + // for a specific aggregate result column. + let mut agg_cols_out = dst + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .collect::>(); + + // Maps each group key to an ordinal offset on output columns. This + // offset is used to update aggregate values for each group key and to + // store the decoded representations of the group keys themselves in + // the associated output columns. + let mut group_keys: HashMap = HashMap::default(); + + // reference back to underlying group columns for fetching decoded group + // key values. + let input_group_columns = dst + .schema + .group_column_names_iter() + .map(|name| self.column_by_name(name)) + .collect::>(); + + let mut next_ordinal_id = 0; // assign a position for each group key in output columns. + for row in 0..total_rows { + // pack each column's encoded value for the row into a packed + // group key. let mut group_key_packed = 0_u128; for (i, col_ids) in groupby_encoded_ids.iter().enumerate() { group_key_packed = pack_u32_in_u128(group_key_packed, col_ids[row], i); } - match groups.raw_entry_mut().from_key(&group_key_packed) { - // aggregates for this group key are already present. Update - // them - hash_map::RawEntryMut::Occupied(mut entry) => { - for (i, values) in aggregate_columns_data.iter().enumerate() { - entry.get_mut()[i].update(values.value(row)); + match group_keys.raw_entry_mut().from_key(&group_key_packed) { + hash_map::RawEntryMut::Occupied(entry) => { + let ordinal_id = entry.get(); + + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + *ordinal_id, + ) } } - // group key does not exist, so create it. hash_map::RawEntryMut::Vacant(entry) => { - let mut group_key_aggs = Vec::with_capacity(dst.schema.aggregate_columns.len()); - for (_, agg_type, _) in &dst.schema.aggregate_columns { - group_key_aggs.push(AggregateResult::from(agg_type)); + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + next_ordinal_id, + ) } - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); + // Add decoded group key values to the output group columns. + for (group_col_i, group_key_col) in group_cols_out.iter_mut().enumerate() { + if group_key_col.len() >= next_ordinal_id { + group_key_col.resize(next_ordinal_id + 1, None); + } + let decoded_value = input_group_columns[group_col_i] + .decode_id(groupby_encoded_ids[group_col_i][row]); + group_key_col[next_ordinal_id] = match decoded_value { + Value::Null => None, + Value::String(s) => Some(s), + _ => panic!("currently unsupported group column"), + }; } - entry.insert(group_key_packed, group_key_aggs); + // update the hashmap with the encoded group key and the + // associated ordinal offset. + entry.insert(group_key_packed, next_ordinal_id); + next_ordinal_id += 1; } } } - // Finally, build results set. Each encoded group key needs to be - // materialised into a logical group key - let columns = dst - .schema - .group_column_names_iter() - .map(|name| self.column_by_name(name)) - .collect::>(); - let mut group_key_vec: Vec> = Vec::with_capacity(groups.len()); - let mut aggregate_vec: Vec> = Vec::with_capacity(groups.len()); - - for (group_key_packed, aggs) in groups.into_iter() { - let mut logical_key = Vec::with_capacity(columns.len()); - - // Unpack the appropriate encoded id for each column from the packed - // group key, then materialise the logical value for that id and add - // it to the materialised group key (`logical_key`). - for (col_idx, column) in columns.iter().enumerate() { - let encoded_id = (group_key_packed >> (col_idx * 32)) as u32; - logical_key.push(column.decode_id(encoded_id)); - } - - group_key_vec.push(GroupKey(logical_key)); - aggregate_vec.push(AggregateResults(aggs)); - } - - dst.group_keys = group_key_vec; - dst.aggregates = aggregate_vec; + dst.group_key_cols = group_cols_out; + dst.aggregate_cols = agg_cols_out; } // Optimised `read_group` method when there are no predicates and all the @@ -1711,6 +1738,9 @@ pub struct ReadAggregateResult<'row_group> { // values for each of the aggregate_columns. pub(crate) aggregates: Vec>, + pub(crate) group_key_cols: Vec>>, + pub(crate) aggregate_cols: Vec>, + pub(crate) group_keys_sorted: bool, } @@ -2657,6 +2687,7 @@ west,POST,304,101,203 ]), ], group_keys_sorted: false, + ..Default::default() }; // Debug implementation diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index e30eebe9fe..e909365147 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -3,7 +3,159 @@ use std::{mem::size_of, sync::Arc}; use arrow_deps::arrow; -use crate::AggregateType; +use crate::{AggregateType, LogicalDataType}; + +#[derive(Clone)] +pub enum AggregateVec<'a> { + Count(Vec), + + SumI64(Vec>), + SumU64(Vec>), + SumF64(Vec>), + + MinU64(Vec>), + MinI64(Vec>), + MinF64(Vec>), + MinString(Vec>), + MinBytes(Vec>), + MinBool(Vec>), + + MaxU64(Vec>), + MaxI64(Vec>), + MaxF64(Vec>), + MaxString(Vec>), + MaxBytes(Vec>), + MaxBool(Vec>), + + FirstU64((Vec>, Vec>)), + FirstI64((Vec>, Vec>)), + FirstF64((Vec>, Vec>)), + FirstString((Vec>, Vec>)), + FirstBytes((Vec>, Vec>)), + FirstBool((Vec>, Vec>)), + + LastU64((Vec>, Vec>)), + LastI64((Vec>, Vec>)), + LastF64((Vec>, Vec>)), + LastString((Vec>, Vec>)), + LastBytes((Vec>, Vec>)), + LastBool((Vec>, Vec>)), +} + +impl AggregateVec<'_> { + pub fn update(&mut self, values: &Values<'_>, row_id: usize, ordinal_position: usize) { + match self { + AggregateVec::Count(arr) => { + if values.is_null(row_id) { + return; + } else if ordinal_position >= arr.len() { + arr.resize(ordinal_position + 1, 0); + } + + arr[ordinal_position] += 1; + } + AggregateVec::SumI64(arr) => { + if values.is_null(row_id) { + return; + } else if ordinal_position >= arr.len() { + arr.resize(ordinal_position + 1, None); + } + + match &mut arr[ordinal_position] { + Some(v) => *v += values.value_i64(row_id), + None => arr[ordinal_position] = Some(values.value_i64(row_id)), + } + } + AggregateVec::SumU64(_) => {} + AggregateVec::SumF64(_) => {} + AggregateVec::MinU64(_) => {} + AggregateVec::MinI64(_) => {} + AggregateVec::MinF64(_) => {} + AggregateVec::MinString(_) => {} + AggregateVec::MinBytes(_) => {} + AggregateVec::MinBool(_) => {} + AggregateVec::MaxU64(_) => {} + AggregateVec::MaxI64(_) => {} + AggregateVec::MaxF64(_) => {} + AggregateVec::MaxString(_) => {} + AggregateVec::MaxBytes(_) => {} + AggregateVec::MaxBool(_) => {} + AggregateVec::FirstU64(_) => {} + AggregateVec::FirstI64(_) => {} + AggregateVec::FirstF64(_) => {} + AggregateVec::FirstString(_) => {} + AggregateVec::FirstBytes(_) => {} + AggregateVec::FirstBool(_) => {} + AggregateVec::LastU64(_) => {} + AggregateVec::LastI64(_) => {} + AggregateVec::LastF64(_) => {} + AggregateVec::LastString(_) => {} + AggregateVec::LastBytes(_) => {} + AggregateVec::LastBool(_) => {} + } + } +} + +impl From<(&AggregateType, &LogicalDataType, usize)> for AggregateVec<'_> { + fn from(v: (&AggregateType, &LogicalDataType, usize)) -> Self { + let length = v.2; + match (v.0, v.1) { + (AggregateType::Count, _) => Self::Count(vec![0; length]), + (AggregateType::First, LogicalDataType::Integer) => { + Self::FirstI64((vec![None; length], vec![None; length])) + } + (AggregateType::First, LogicalDataType::Unsigned) => { + Self::FirstU64((vec![None; length], vec![None; length])) + } + (AggregateType::First, LogicalDataType::Float) => { + Self::FirstF64((vec![None; length], vec![None; length])) + } + (AggregateType::First, LogicalDataType::String) => { + Self::FirstString((vec![None; length], vec![None; length])) + } + (AggregateType::First, LogicalDataType::Binary) => { + Self::FirstBytes((vec![None; length], vec![None; length])) + } + (AggregateType::First, LogicalDataType::Boolean) => { + Self::FirstBool((vec![None; length], vec![None; length])) + } + (AggregateType::Last, LogicalDataType::Integer) => { + Self::LastI64((vec![None; length], vec![None; length])) + } + (AggregateType::Last, LogicalDataType::Unsigned) => { + Self::LastU64((vec![None; length], vec![None; length])) + } + (AggregateType::Last, LogicalDataType::Float) => { + Self::LastF64((vec![None; length], vec![None; length])) + } + (AggregateType::Last, LogicalDataType::String) => { + Self::LastString((vec![None; length], vec![None; length])) + } + (AggregateType::Last, LogicalDataType::Binary) => { + Self::LastBytes((vec![None; length], vec![None; length])) + } + (AggregateType::Last, LogicalDataType::Boolean) => { + Self::LastBool((vec![None; length], vec![None; length])) + } + (AggregateType::Min, LogicalDataType::Integer) => Self::MinI64(vec![None; length]), + (AggregateType::Min, LogicalDataType::Unsigned) => Self::MinU64(vec![None; length]), + (AggregateType::Min, LogicalDataType::Float) => Self::MinF64(vec![None; length]), + (AggregateType::Min, LogicalDataType::String) => Self::MinString(vec![None; length]), + (AggregateType::Min, LogicalDataType::Binary) => Self::MinBytes(vec![None; length]), + (AggregateType::Min, LogicalDataType::Boolean) => Self::MinBool(vec![None; length]), + (AggregateType::Max, LogicalDataType::Integer) => Self::MaxI64(vec![None; length]), + (AggregateType::Max, LogicalDataType::Unsigned) => Self::MaxU64(vec![None; length]), + (AggregateType::Max, LogicalDataType::Float) => Self::MaxF64(vec![None; length]), + (AggregateType::Max, LogicalDataType::String) => Self::MaxString(vec![None; length]), + (AggregateType::Max, LogicalDataType::Binary) => Self::MaxBytes(vec![None; length]), + (AggregateType::Max, LogicalDataType::Boolean) => Self::MaxBool(vec![None; length]), + (AggregateType::Sum, LogicalDataType::Integer) => Self::SumI64(vec![None; length]), + (AggregateType::Sum, LogicalDataType::Unsigned) => Self::SumU64(vec![None; length]), + (AggregateType::Sum, LogicalDataType::Float) => Self::SumF64(vec![None; length]), + (AggregateType::Sum, _) => unreachable!("unsupported SUM aggregates"), + } + } +} /// These variants hold aggregates, which are the results of applying aggregates /// to column data. @@ -659,6 +811,20 @@ impl<'a> Values<'a> { self.len() == 0 } + pub fn is_null(&self, i: usize) -> bool { + match &self { + Self::String(c) => c[i].is_none(), + Self::F64(_) => false, + Self::I64(_) => false, + Self::U64(_) => false, + Self::Bool(c) => c[i].is_none(), + Self::ByteArray(c) => c[i].is_none(), + Self::I64N(c) => c[i].is_none(), + Self::U64N(c) => c[i].is_none(), + Self::F64N(c) => c[i].is_none(), + } + } + pub fn value(&self, i: usize) -> Value<'a> { match &self { Self::String(c) => match c[i] { @@ -690,6 +856,15 @@ impl<'a> Values<'a> { }, } } + + // Returns a value as an i64. Panics if not possible. + fn value_i64(&self, i: usize) -> i64 { + match &self { + Values::I64(c) => c[i], + Values::I64N(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as i64"), + } + } } /// Moves ownership of Values into an arrow `ArrayRef`. From fd7cdc4fa4b643597d2f5a71aa56498d93f9664f Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Fri, 12 Mar 2021 18:15:22 +0000 Subject: [PATCH 079/104] refactor: remove single column group_by --- read_buffer/src/row_group.rs | 189 ++++++++++++++--------------------- 1 file changed, 73 insertions(+), 116 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index d015648f93..88df3f6151 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -494,17 +494,6 @@ impl RowGroup { aggregate_columns_data.push(column_values); } - // If there is a single group column then we can use an optimised - // approach for building group keys - if group_columns.len() == 1 { - self.read_group_single_group_column( - &mut result, - &groupby_encoded_ids[0], - aggregate_columns_data, - ); - return result; - } - // Perform the group by using a hashmap self.read_group_with_hashing(&mut result, &groupby_encoded_ids, aggregate_columns_data); result @@ -539,71 +528,102 @@ impl RowGroup { &'a self, dst: &mut ReadAggregateResult<'a>, groupby_encoded_ids: &[Vec], - aggregate_columns_data: &[Values<'a>], + aggregate_input_columns: &[Values<'a>], ) { - // Now begin building the group keys. - let mut groups: HashMap, Vec>> = HashMap::default(); let total_rows = groupby_encoded_ids[0].len(); assert!(groupby_encoded_ids.iter().all(|x| x.len() == total_rows)); - // key_buf will be used as a temporary buffer for group keys, which are - // themselves integers. - let mut key_buf = vec![0; dst.schema.group_columns.len()]; + // These vectors will hold the decoded values of each part of each + // group key. They are the output columns of the input columns used for + // the grouping operation. + let mut group_cols_out: Vec>>> = vec![]; + group_cols_out.resize(groupby_encoded_ids.len(), vec![]); + // Each of these vectors will be used to store each aggregate row-value + // for a specific aggregate result column. + let mut agg_cols_out = dst + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .collect::>(); + + // Maps each group key to an ordinal offset on output columns. This + // offset is used to update aggregate values for each group key and to + // store the decoded representations of the group keys themselves in + // the associated output columns. + let mut group_keys: HashMap, usize> = HashMap::default(); + + // reference back to underlying group columns for fetching decoded group + // key values. + let input_group_columns = dst + .schema + .group_column_names_iter() + .map(|name| self.column_by_name(name)) + .collect::>(); + + // key_buf will be used as a temporary buffer for group keys represented + // as a `Vec`. + let mut key_buf = vec![0; dst.schema.group_columns.len()]; + let mut next_ordinal_id = 0; // assign a position for each group key in output columns. for row in 0..total_rows { // update the group key buffer with the group key for this row for (j, col_ids) in groupby_encoded_ids.iter().enumerate() { key_buf[j] = col_ids[row]; } - match groups.raw_entry_mut().from_key(&key_buf) { - // aggregates for this group key are already present. Update - // them - hash_map::RawEntryMut::Occupied(mut entry) => { - for (i, values) in aggregate_columns_data.iter().enumerate() { - entry.get_mut()[i].update(values.value(row)); + match group_keys.raw_entry_mut().from_key(&key_buf) { + hash_map::RawEntryMut::Occupied(entry) => { + let ordinal_id = entry.get(); + + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + *ordinal_id, + ) } } // group key does not exist, so create it. hash_map::RawEntryMut::Vacant(entry) => { - let mut group_key_aggs = Vec::with_capacity(dst.schema.aggregate_columns.len()); - for (_, agg_type, _) in &dst.schema.aggregate_columns { - group_key_aggs.push(AggregateResult::from(agg_type)); + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + next_ordinal_id, + ) } - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); + // Add decoded group key values to the output group columns. + for (group_col_i, group_key_col) in group_cols_out.iter_mut().enumerate() { + if group_key_col.len() >= next_ordinal_id { + group_key_col.resize(next_ordinal_id + 1, None); + } + let decoded_value = input_group_columns[group_col_i] + .decode_id(groupby_encoded_ids[group_col_i][row]); + group_key_col[next_ordinal_id] = match decoded_value { + Value::Null => None, + Value::String(s) => Some(s), + _ => panic!("currently unsupported group column"), + }; } - entry.insert(key_buf.clone(), group_key_aggs); + // update the hashmap with the encoded group key and the + // associated ordinal offset. + entry.insert(key_buf.clone(), next_ordinal_id); + next_ordinal_id += 1; } } } - // Finally, build results set. Each encoded group key needs to be - // materialised into a logical group key - let columns = dst - .schema - .group_column_names_iter() - .map(|name| self.column_by_name(name)) - .collect::>(); - let mut group_key_vec: Vec> = Vec::with_capacity(groups.len()); - let mut aggregate_vec: Vec> = Vec::with_capacity(groups.len()); - - for (group_key, aggs) in groups.into_iter() { - let mut logical_key = Vec::with_capacity(group_key.len()); - for (col_idx, &encoded_id) in group_key.iter().enumerate() { - // TODO(edd): address the cast to u32 - logical_key.push(columns[col_idx].decode_id(encoded_id as u32)); - } - - group_key_vec.push(GroupKey(logical_key)); - aggregate_vec.push(AggregateResults(aggs)); - } - - // update results - dst.group_keys = group_key_vec; - dst.aggregates = aggregate_vec; + dst.group_key_cols = group_cols_out; + dst.aggregate_cols = agg_cols_out; } // This function is similar to `read_group_hash_with_vec_key` in that it @@ -830,69 +850,6 @@ impl RowGroup { } } - // Optimised `read_group` path for queries where only a single column is - // being grouped on. In this case building a hash table is not necessary, - // and the group keys can be used as indexes into a vector whose values - // contain aggregates. As rows are processed these aggregates can be updated - // in constant time. - fn read_group_single_group_column<'a>( - &'a self, - dst: &mut ReadAggregateResult<'a>, - groupby_encoded_ids: &[u32], - aggregate_columns_data: Vec>, - ) { - assert_eq!(dst.schema().group_columns.len(), 1); - let column = self.column_by_name(dst.schema.group_column_names_iter().next().unwrap()); - - // Allocate a vector to hold aggregates that can be updated as rows are - // processed. An extra group is required because encoded ids are - // 0-indexed. - let required_groups = groupby_encoded_ids.iter().max().unwrap() + 1; - let mut groups: Vec>>> = - vec![None; required_groups as usize]; - - for (row, encoded_id) in groupby_encoded_ids.iter().enumerate() { - let idx = *encoded_id as usize; - match &mut groups[idx] { - Some(group_key_aggs) => { - // Update all aggregates for the group key - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); - } - } - None => { - let mut group_key_aggs = dst - .schema - .aggregate_columns - .iter() - .map(|(_, agg_type, _)| AggregateResult::from(agg_type)) - .collect::>(); - - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); - } - - groups[idx] = Some(group_key_aggs); - } - } - } - - // Finally, build results set. Each encoded group key needs to be - // materialised into a logical group key - let mut group_key_vec: Vec> = Vec::with_capacity(groups.len()); - let mut aggregate_vec = Vec::with_capacity(groups.len()); - - for (group_key, aggs) in groups.into_iter().enumerate() { - if let Some(aggs) = aggs { - group_key_vec.push(GroupKey(vec![column.decode_id(group_key as u32)])); - aggregate_vec.push(AggregateResults(aggs)); - } - } - - dst.group_keys = group_key_vec; - dst.aggregates = aggregate_vec; - } - // Optimised `read_group` method for cases where the columns being grouped // are already totally ordered in the `RowGroup`. // From 3725161fc3a035ee33e586337d0f0cc297559c19 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Tue, 16 Mar 2021 10:41:28 +0000 Subject: [PATCH 080/104] refactor: use AggregateVec on rle grouping --- read_buffer/src/row_group.rs | 115 +++++++++++++++++++++-------------- read_buffer/src/value.rs | 65 +++++++++++++++++--- 2 files changed, 127 insertions(+), 53 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index 88df3f6151..e2e9e6f6cd 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -644,8 +644,8 @@ impl RowGroup { assert!(dst.schema.group_columns.len() <= 4); // These vectors will hold the decoded values of each part of each - // group key. They are the output columns of the input columns used for - // the grouping operation. + // group key. They are the output columns derived from the input + // grouping columns. let mut group_cols_out: Vec>>> = vec![]; group_cols_out.resize(groupby_encoded_ids.len(), vec![]); @@ -740,20 +740,24 @@ impl RowGroup { // In this case all the grouping columns pre-computed bitsets for each // distinct value. fn read_group_all_rows_all_rle<'a>(&'a self, dst: &mut ReadAggregateResult<'a>) { - let group_columns = dst + // References to the columns to be used as input for producing the + // output aggregates. + let input_group_columns = dst .schema .group_column_names_iter() .map(|name| self.column_by_name(name)) .collect::>(); - let aggregate_columns_typ = dst + // References to the columns to be used as input for producing the + // output aggregates. Also returns the required aggregate type. + let input_aggregate_columns = dst .schema .aggregate_columns .iter() .map(|(col_type, agg_type, _)| (self.column_by_name(col_type.as_str()), *agg_type)) .collect::>(); - let encoded_groups = dst + let groupby_encoded_ids = dst .schema .group_column_names_iter() .map(|col_type| { @@ -763,6 +767,21 @@ impl RowGroup { }) .collect::>(); + // These vectors will hold the decoded values of each part of each + // group key. They are the output columns derived from the input + // grouping columns. + let mut group_cols_out: Vec>>> = vec![]; + group_cols_out.resize(groupby_encoded_ids.len(), vec![]); + + // Each of these vectors will be used to store each aggregate row-value + // for a specific aggregate result column. + let mut agg_cols_out = dst + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .collect::>(); + // multi_cartesian_product will create the cartesian product of all // grouping-column values. This is likely going to be more group keys // than there exists row-data for, so don't materialise them yet... @@ -788,66 +807,71 @@ impl RowGroup { // [0, 3], [1, 3], [2, 3], [2, 4], [3, 2], [4, 1] // // We figure out which group keys have data and which don't in the loop - // below, by intersecting bitsets for each id and checking for non-empty - // sets. - let group_keys = encoded_groups + // below, by intersecting row_id bitsets for each encoded id, and + // checking for non-empty sets. + let candidate_group_keys = groupby_encoded_ids .iter() .map(|ids| (0..ids.len())) .multi_cartesian_product(); // Let's figure out which of the candidate group keys are actually group // keys with data. - 'outer: for group_key in group_keys { - let mut aggregate_row_ids = - Cow::Borrowed(encoded_groups[0][group_key[0]].unwrap_bitmap()); + 'outer: for group_key_buf in candidate_group_keys { + let mut group_key_row_ids = + Cow::Borrowed(groupby_encoded_ids[0][group_key_buf[0]].unwrap_bitmap()); - if aggregate_row_ids.is_empty() { + if group_key_row_ids.is_empty() { continue; } - for i in 1..group_key.len() { - let other = encoded_groups[i][group_key[i]].unwrap_bitmap(); + for i in 1..group_key_buf.len() { + let other = groupby_encoded_ids[i][group_key_buf[i]].unwrap_bitmap(); - if aggregate_row_ids.and_cardinality(other) > 0 { - aggregate_row_ids = Cow::Owned(aggregate_row_ids.and(other)); + if group_key_row_ids.and_cardinality(other) > 0 { + group_key_row_ids = Cow::Owned(group_key_row_ids.and(other)); } else { continue 'outer; } } - // This group key has some matching row ids. Materialise the group - // key and calculate the aggregates. + // There exist rows for this group key combination. Materialise the + // group key and calculate the aggregates for this key using set + // of row IDs. - // TODO(edd): given these RLE columns should have low cardinality - // there should be a reasonably low group key cardinality. It could - // be safe to use `small_vec` here without blowing the stack up. - let mut material_key = Vec::with_capacity(group_key.len()); - for (col_idx, &encoded_id) in group_key.iter().enumerate() { - material_key.push(group_columns[col_idx].decode_id(encoded_id as u32)); - } - dst.group_keys.push(GroupKey(material_key)); + // Add decoded group key values to the output group columns. + for (group_col_i, col) in group_cols_out.iter_mut().enumerate() { + let decoded_value = + input_group_columns[group_col_i].decode_id(group_key_buf[group_col_i] as u32); - let mut aggregates = Vec::with_capacity(aggregate_columns_typ.len()); - for (agg_col, typ) in &aggregate_columns_typ { - aggregates.push(match typ { - AggregateType::Count => { - AggregateResult::Count(agg_col.count(&aggregate_row_ids.to_vec()) as u64) - } - AggregateType::First => todo!(), - AggregateType::Last => todo!(), - AggregateType::Min => { - AggregateResult::Min(agg_col.min(&aggregate_row_ids.to_vec())) - } - AggregateType::Max => { - AggregateResult::Max(agg_col.max(&aggregate_row_ids.to_vec())) - } - AggregateType::Sum => { - AggregateResult::Sum(agg_col.sum(&aggregate_row_ids.to_vec())) - } + col.push(match decoded_value { + Value::Null => None, + Value::String(s) => Some(s), + _ => panic!("currently unsupported group column"), }); } - dst.aggregates.push(AggregateResults(aggregates)); + + // Calculate an aggregate from each input aggregate column and + // set it at the relevant offset in the output column. + for (agg_col_i, (agg_col, typ)) in input_aggregate_columns.iter().enumerate() { + match typ { + AggregateType::Count => { + let agg = agg_col.count(&group_key_row_ids.to_vec()) as u64; + agg_cols_out[agg_col_i].push(&Value::Scalar(Scalar::U64(agg))) + } + AggregateType::First => {} + AggregateType::Last => {} + AggregateType::Min => {} + AggregateType::Max => {} + AggregateType::Sum => { + let agg = agg_col.sum(&group_key_row_ids.to_vec()); + agg_cols_out[agg_col_i].push(&Value::Scalar(agg)); + } + } + } } + + dst.group_key_cols = group_cols_out; + dst.aggregate_cols = agg_cols_out; } // Optimised `read_group` method for cases where the columns being grouped @@ -908,7 +932,8 @@ impl RowGroup { AggregateType::Max => { aggregate_row.push(AggregateResult::Max(col.max(&row_ids))); } - _ => unimplemented!("Other aggregates are not yet supported"), + AggregateType::First => unimplemented!("First not yet implemented"), + AggregateType::Last => unimplemented!("Last not yet implemented"), } } dst.aggregates.push(AggregateResults(aggregate_row)); // write the row diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index e909365147..94ecdd8cb1 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -43,27 +43,27 @@ pub enum AggregateVec<'a> { } impl AggregateVec<'_> { - pub fn update(&mut self, values: &Values<'_>, row_id: usize, ordinal_position: usize) { + pub fn update(&mut self, values: &Values<'_>, row_id: usize, offset: usize) { match self { AggregateVec::Count(arr) => { if values.is_null(row_id) { return; - } else if ordinal_position >= arr.len() { - arr.resize(ordinal_position + 1, 0); + } else if offset >= arr.len() { + arr.resize(offset + 1, 0); } - arr[ordinal_position] += 1; + arr[offset] += 1; } AggregateVec::SumI64(arr) => { if values.is_null(row_id) { return; - } else if ordinal_position >= arr.len() { - arr.resize(ordinal_position + 1, None); + } else if offset >= arr.len() { + arr.resize(offset + 1, None); } - match &mut arr[ordinal_position] { + match &mut arr[offset] { Some(v) => *v += values.value_i64(row_id), - None => arr[ordinal_position] = Some(values.value_i64(row_id)), + None => arr[offset] = Some(values.value_i64(row_id)), } } AggregateVec::SumU64(_) => {} @@ -94,6 +94,41 @@ impl AggregateVec<'_> { AggregateVec::LastBool(_) => {} } } + + /// Appends the provided value to the end of the aggregate vector. + /// Panics if the type of `Value` does not satisfy the aggregate type. + pub fn push(&mut self, value: &Value<'_>) { + match self { + AggregateVec::Count(arr) => arr.push(value.u64()), + AggregateVec::SumI64(arr) => arr.push(Some(value.i64())), + AggregateVec::SumU64(_) => {} + AggregateVec::SumF64(_) => {} + AggregateVec::MinU64(_) => {} + AggregateVec::MinI64(_) => {} + AggregateVec::MinF64(_) => {} + AggregateVec::MinString(_) => {} + AggregateVec::MinBytes(_) => {} + AggregateVec::MinBool(_) => {} + AggregateVec::MaxU64(_) => {} + AggregateVec::MaxI64(_) => {} + AggregateVec::MaxF64(_) => {} + AggregateVec::MaxString(_) => {} + AggregateVec::MaxBytes(_) => {} + AggregateVec::MaxBool(_) => {} + AggregateVec::FirstU64(_) => {} + AggregateVec::FirstI64(_) => {} + AggregateVec::FirstF64(_) => {} + AggregateVec::FirstString(_) => {} + AggregateVec::FirstBytes(_) => {} + AggregateVec::FirstBool(_) => {} + AggregateVec::LastU64(_) => {} + AggregateVec::LastI64(_) => {} + AggregateVec::LastF64(_) => {} + AggregateVec::LastString(_) => {} + AggregateVec::LastBytes(_) => {} + AggregateVec::LastBool(_) => {} + } + } } impl From<(&AggregateType, &LogicalDataType, usize)> for AggregateVec<'_> { @@ -705,6 +740,20 @@ impl Value<'_> { panic!("cannot unwrap Value to Scalar"); } + pub fn i64(&self) -> i64 { + if let Self::Scalar(Scalar::I64(v)) = self { + return *v; + } + panic!("cannot unwrap Value to i64"); + } + + pub fn u64(&self) -> u64 { + if let Self::Scalar(Scalar::U64(v)) = self { + return *v; + } + panic!("cannot unwrap Value to u64"); + } + pub fn string(&self) -> &str { if let Self::String(s) = self { return s; From e5f7cc143ab700c20c29b8a3d4590cfe30b0059e Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Thu, 18 Mar 2021 17:35:33 +0000 Subject: [PATCH 081/104] refactor: use AggregateVec for results --- read_buffer/src/column.rs | 2 +- read_buffer/src/row_group.rs | 837 ++++++++++++++----------- read_buffer/src/schema.rs | 1 + read_buffer/src/table.rs | 22 +- read_buffer/src/value.rs | 1144 +++++++++++++++++++++++++++++++--- 5 files changed, 1551 insertions(+), 455 deletions(-) diff --git a/read_buffer/src/column.rs b/read_buffer/src/column.rs index 8bf66bddc1..7260754dc1 100644 --- a/read_buffer/src/column.rs +++ b/read_buffer/src/column.rs @@ -330,7 +330,7 @@ impl Column { // Check the column for all rows that satisfy the predicate. let row_ids = match &self { - Self::String(_, data) => data.row_ids_filter(op, value.string(), dst), + Self::String(_, data) => data.row_ids_filter(op, value.str(), dst), Self::Float(_, data) => data.row_ids_filter(op, value.scalar(), dst), Self::Integer(_, data) => data.row_ids_filter(op, value.scalar(), dst), Self::Unsigned(_, data) => data.row_ids_filter(op, value.scalar(), dst), diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index e2e9e6f6cd..db4bdf7b11 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -514,11 +514,11 @@ impl RowGroup { // single 128-bit integer as the group key. If grouping is on more than // four columns then a fallback to using an vector as a key will happen. if dst.schema.group_columns.len() <= 4 { - self.read_group_hash_with_u128_key(dst, &groupby_encoded_ids, &aggregate_columns_data); + self.read_group_hash_with_u128_key(dst, &groupby_encoded_ids, aggregate_columns_data); return; } - self.read_group_hash_with_vec_key(dst, &groupby_encoded_ids, &aggregate_columns_data); + self.read_group_hash_with_vec_key(dst, &groupby_encoded_ids, aggregate_columns_data); } // This function is used with `read_group_hash` when the number of columns @@ -528,7 +528,7 @@ impl RowGroup { &'a self, dst: &mut ReadAggregateResult<'a>, groupby_encoded_ids: &[Vec], - aggregate_input_columns: &[Values<'a>], + aggregate_input_columns: Vec>, ) { let total_rows = groupby_encoded_ids[0].len(); assert!(groupby_encoded_ids.iter().all(|x| x.len() == total_rows)); @@ -637,7 +637,7 @@ impl RowGroup { &'a self, dst: &mut ReadAggregateResult<'a>, groupby_encoded_ids: &[Vec], - aggregate_input_columns: &[Values<'a>], + aggregate_input_columns: Vec>, ) { let total_rows = groupby_encoded_ids[0].len(); assert!(groupby_encoded_ids.iter().all(|x| x.len() == total_rows)); @@ -782,6 +782,8 @@ impl RowGroup { .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) .collect::>(); + let mut output_rows = 0; + // multi_cartesian_product will create the cartesian product of all // grouping-column values. This is likely going to be more group keys // than there exists row-data for, so don't materialise them yet... @@ -837,6 +839,7 @@ impl RowGroup { // There exist rows for this group key combination. Materialise the // group key and calculate the aggregates for this key using set // of row IDs. + output_rows += 1; // Add decoded group key values to the output group columns. for (group_col_i, col) in group_cols_out.iter_mut().enumerate() { @@ -856,20 +859,33 @@ impl RowGroup { match typ { AggregateType::Count => { let agg = agg_col.count(&group_key_row_ids.to_vec()) as u64; - agg_cols_out[agg_col_i].push(&Value::Scalar(Scalar::U64(agg))) + agg_cols_out[agg_col_i].push(Value::Scalar(Scalar::U64(agg))) } AggregateType::First => {} AggregateType::Last => {} - AggregateType::Min => {} - AggregateType::Max => {} + AggregateType::Min => { + let agg = agg_col.min(&group_key_row_ids.to_vec()); + agg_cols_out[agg_col_i].push(agg); + } + AggregateType::Max => { + let agg = agg_col.max(&group_key_row_ids.to_vec()); + agg_cols_out[agg_col_i].push(agg); + } AggregateType::Sum => { let agg = agg_col.sum(&group_key_row_ids.to_vec()); - agg_cols_out[agg_col_i].push(&Value::Scalar(agg)); + agg_cols_out[agg_col_i].push(Value::Scalar(agg)); } } } } + for col in &group_cols_out { + assert_eq!(col.len(), output_rows); + } + for col in &agg_cols_out { + assert_eq!(col.len(), output_rows); + } + dst.group_key_cols = group_cols_out; dst.aggregate_cols = agg_cols_out; } @@ -890,13 +906,6 @@ impl RowGroup { // Applies aggregates on multiple columns with an optional predicate. fn aggregate_columns<'a>(&'a self, predicate: &Predicate, dst: &mut ReadAggregateResult<'a>) { - let aggregate_columns = dst - .schema - .aggregate_columns - .iter() - .map(|(col_type, agg_type, _)| (self.column_by_name(col_type.as_str()), *agg_type)) - .collect::>(); - let row_ids = match predicate.is_empty() { true => { // TODO(edd): PERF - teach each column encoding how to produce @@ -916,27 +925,38 @@ impl RowGroup { }, }; - // the single row that will store the aggregate column values. - let mut aggregate_row = vec![]; - for (col, agg_type) in aggregate_columns { + // References to the columns to be used as input for producing the + // output aggregates. Also returns the required aggregate type. + let input_aggregate_columns = dst + .schema + .aggregate_columns + .iter() + .map(|(col_type, agg_type, _)| (self.column_by_name(col_type.as_str()), *agg_type)) + .collect::>(); + + let mut output_aggregate_columns = dst + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .collect::>(); + + for (i, (col, agg_type)) in input_aggregate_columns.iter().enumerate() { match agg_type { AggregateType::Count => { - aggregate_row.push(AggregateResult::Count(col.count(&row_ids) as u64)); - } - AggregateType::Sum => { - aggregate_row.push(AggregateResult::Sum(col.sum(&row_ids))); - } - AggregateType::Min => { - aggregate_row.push(AggregateResult::Min(col.min(&row_ids))); - } - AggregateType::Max => { - aggregate_row.push(AggregateResult::Max(col.max(&row_ids))); + let value = Value::Scalar(Scalar::U64(col.count(&row_ids) as u64)); + output_aggregate_columns[i].push(value); } AggregateType::First => unimplemented!("First not yet implemented"), AggregateType::Last => unimplemented!("Last not yet implemented"), + AggregateType::Min => output_aggregate_columns[i].push(col.min(&row_ids)), + AggregateType::Max => output_aggregate_columns[i].push(col.max(&row_ids)), + AggregateType::Sum => { + output_aggregate_columns[i].push(Value::Scalar(col.sum(&row_ids))) + } } } - dst.aggregates.push(AggregateResults(aggregate_row)); // write the row + dst.aggregate_cols = output_aggregate_columns; } /// Given the predicate (which may be empty), determine a set of rows @@ -1353,58 +1373,6 @@ impl TryFrom<&DfExpr> for BinaryExpr { } } -// A GroupKey is an ordered collection of row values. The order determines which -// columns the values originated from. -#[derive(PartialEq, PartialOrd, Clone)] -pub struct GroupKey<'row_group>(Vec>); - -impl GroupKey<'_> { - fn len(&self) -> usize { - self.0.len() - } -} - -impl<'a> From>> for GroupKey<'a> { - fn from(values: Vec>) -> Self { - Self(values) - } -} - -impl Eq for GroupKey<'_> {} - -// Implementing the `Ord` trait on `GroupKey` means that collections of group -// keys become sortable. This is typically useful for test because depending on -// the implementation, group keys are not always emitted in sorted order. -// -// To be compared, group keys *must* have the same length, or `cmp` will panic. -// They will be ordered as follows: -// -// [foo, zoo, zoo], [foo, bar, zoo], [bar, bar, bar], [bar, bar, zoo], -// -// becomes: -// -// [bar, bar, bar], [bar, bar, zoo], [foo, bar, zoo], [foo, zoo, zoo], -// -// Be careful sorting group keys in result sets, because other columns -// associated with the group keys won't be sorted unless the correct `sort` -// methods are used on the result set implementations. -impl Ord for GroupKey<'_> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // two group keys must have same length - assert_eq!(self.0.len(), other.0.len()); - - let cols = self.0.len(); - for i in 0..cols { - match self.0[i].partial_cmp(&other.0[i]) { - Some(ord) => return ord, - None => continue, - } - } - - std::cmp::Ordering::Equal - } -} - #[derive(Debug, PartialEq, Clone)] pub struct AggregateResults<'row_group>(Vec>); @@ -1712,61 +1680,68 @@ pub struct ReadAggregateResult<'row_group> { // a schema describing the columns in the results and their types. pub(crate) schema: ResultSchema, - // row-wise collection of group keys. Each group key contains column-wise - // values for each of the groupby_columns. - pub(crate) group_keys: Vec>, - - // row-wise collection of aggregates. Each aggregate contains column-wise - // values for each of the aggregate_columns. - pub(crate) aggregates: Vec>, - + // The collection of columns forming the group keys. pub(crate) group_key_cols: Vec>>, - pub(crate) aggregate_cols: Vec>, + + // The collection of aggregate columns. Each value in each column is an + // aggregate associated with the group key built from values in the group + // columns and the same ordinal position. + pub(crate) aggregate_cols: Vec, pub(crate) group_keys_sorted: bool, } impl<'row_group> ReadAggregateResult<'row_group> { - fn with_capacity(schema: ResultSchema, capacity: usize) -> Self { + pub fn new(schema: ResultSchema) -> Self { Self { schema, - group_keys: Vec::with_capacity(capacity), - aggregates: Vec::with_capacity(capacity), ..Default::default() } } + /// A `ReadAggregateResult` is empty if there are no aggregate columns. pub fn is_empty(&self) -> bool { - self.aggregates.is_empty() + self.aggregate_cols.is_empty() } pub fn schema(&self) -> &ResultSchema { &self.schema } - // The number of rows in the result. + //9/ The number of rows in the result. pub fn rows(&self) -> usize { - self.aggregates.len() + if self.aggregate_cols.is_empty() { + return 0; + } + self.aggregate_cols[0].len() } - // The number of distinct group keys in the result. + // The number of distinct group keys in the result. Not the same as `rows()` + // because a `ReadAggregateResult` can have no group keys and have a single + // aggregate row. pub fn cardinality(&self) -> usize { - self.group_keys.len() + if self.group_key_cols.is_empty() { + return 0; + } + self.group_key_cols[0].len() } // Is this result for a grouped aggregate? pub fn is_grouped_aggregate(&self) -> bool { - !self.group_keys.is_empty() + !self.group_key_cols.is_empty() + } + + // The number of grouping columns. + pub fn group_key_columns(&self) -> usize { + self.group_key_cols.len() } // Whether or not the rows in the results are sorted by group keys or not. pub fn group_keys_sorted(&self) -> bool { - self.group_keys.is_empty() || self.group_keys_sorted + self.group_key_cols.is_empty() || self.group_keys_sorted } /// Merges `other` and self, returning a new set of results. - /// - /// NOTE: This is slow! Not expected to be the final type of implementation pub fn merge( mut self, mut other: ReadAggregateResult<'row_group>, @@ -1790,194 +1765,355 @@ impl<'row_group> ReadAggregateResult<'row_group> { other.sort(); } - let self_group_keys = self.cardinality(); - let self_len = self.rows(); - let other_len = other.rows(); - let mut result = Self::with_capacity(self.schema, self_len.max(other_len)); + let mut result = Self::new(self.schema.clone()); + // Allocate output grouping columns + result + .group_key_cols + .resize(result.schema.group_columns.len(), vec![]); + + // Allocate output aggregate columns + result.aggregate_cols = result + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .collect::>(); + + let mut self_i = 0; + let mut other_i = 0; + while self_i < self.rows() || other_i < other.rows() { + if self_i == self.rows() { + // drained self, add the rest of other's group key columns + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.extend(other.group_key_cols[col_i].iter().skip(other_i)); + } + + // add the rest of other's aggregate columns + // + // N.B - by checking the data type of the aggregate columns here + // we can do type checking on a column basis (once) rather than + // for each row. This allows us to extract an aggregate vec + // and an iterator of the same type to extend the aggregate vec. + for (col_i, (_, _, data_type)) in result.schema.aggregate_columns.iter().enumerate() + { + match data_type { + LogicalDataType::Integer => { + let itr = other.aggregate_cols[col_i].as_i64().iter().cloned(); + result.aggregate_cols[col_i].extend_with_i64(itr); + } + LogicalDataType::Unsigned => { + let itr = other.aggregate_cols[col_i].as_u64().iter().cloned(); + result.aggregate_cols[col_i].extend_with_u64(itr); + } + LogicalDataType::Float => { + let itr = other.aggregate_cols[col_i].as_f64().iter().cloned(); + result.aggregate_cols[col_i].extend_with_f64(itr); + } + LogicalDataType::String => { + let itr = other.aggregate_cols[col_i].as_str().iter().cloned(); + result.aggregate_cols[col_i].extend_with_str(itr); + } + LogicalDataType::Binary => { + let itr = other.aggregate_cols[col_i].as_bytes().iter().cloned(); + result.aggregate_cols[col_i].extend_with_bytes(itr); + } + LogicalDataType::Boolean => { + let itr = other.aggregate_cols[col_i].as_bool().iter().cloned(); + result.aggregate_cols[col_i].extend_with_bool(itr); + } + } + } - let mut i: usize = 0; - let mut j: usize = 0; - while i < self_len || j < other_len { - if i >= self_len { - // drained self, add the rest of other - result - .group_keys - .extend(other.group_keys.iter().skip(j).cloned()); - result - .aggregates - .extend(other.aggregates.iter().skip(j).cloned()); return result; - } else if j >= other_len { - // drained other, add the rest of self - result - .group_keys - .extend(self.group_keys.iter().skip(j).cloned()); - result - .aggregates - .extend(self.aggregates.iter().skip(j).cloned()); + } else if other_i == other.rows() { + // drained other, add the rest of self's group key columns + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.extend(self.group_key_cols[col_i].iter().skip(self_i)); + } + + // add the rest of self's aggregate columns + for (col_i, (_, _, data_type)) in result.schema.aggregate_columns.iter().enumerate() + { + match data_type { + LogicalDataType::Integer => { + let itr = self.aggregate_cols[col_i].as_i64().iter().cloned(); + result.aggregate_cols[col_i].extend_with_i64(itr); + } + LogicalDataType::Unsigned => { + let itr = self.aggregate_cols[col_i].as_u64().iter().cloned(); + result.aggregate_cols[col_i].extend_with_u64(itr); + } + LogicalDataType::Float => { + let itr = self.aggregate_cols[col_i].as_f64().iter().cloned(); + result.aggregate_cols[col_i].extend_with_f64(itr); + } + LogicalDataType::String => { + let itr = self.aggregate_cols[col_i].as_str().iter().cloned(); + result.aggregate_cols[col_i].extend_with_str(itr); + } + LogicalDataType::Binary => { + let itr = self.aggregate_cols[col_i].as_bytes().iter().cloned(); + result.aggregate_cols[col_i].extend_with_bytes(itr); + } + LogicalDataType::Boolean => { + let itr = self.aggregate_cols[col_i].as_bool().iter().cloned(); + result.aggregate_cols[col_i].extend_with_bool(itr); + } + } + } + return result; } - // just merge the aggregate if there are no group keys - if self_group_keys == 0 { - assert!((self_len == other_len) && self_len == 1); // there should be a single aggregate row - self.aggregates[i].merge(&other.aggregates[j]); - result.aggregates.push(self.aggregates[i].clone()); - return result; + // compare the next row in self and other and determine if there is + // a clear lexicographic order. + let mut ord = Ordering::Equal; + for i in 0..result.schema.group_columns.len() { + match self.group_key_cols[i][self_i].partial_cmp(&other.group_key_cols[i][other_i]) + { + Some(o) => { + ord = o; + if !matches!(ord, Ordering::Equal) { + break; + } + } + None => continue, + } } - // there are group keys so merge them. - match self.group_keys[i].cmp(&other.group_keys[j]) { + match ord { Ordering::Less => { - result.group_keys.push(self.group_keys[i].clone()); - result.aggregates.push(self.aggregates[i].clone()); - i += 1; + // move the next row for each of self's columns onto result. + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.push(self.group_key_cols[col_i][self_i]); + } + for (col_i, col) in result.aggregate_cols.iter_mut().enumerate() { + col.push(self.aggregate_cols[col_i].value(self_i)); + } + self_i += 1; } Ordering::Equal => { - // merge aggregates - self.aggregates[i].merge(&other.aggregates[j]); - result.group_keys.push(self.group_keys[i].clone()); - result.aggregates.push(self.aggregates[i].clone()); - i += 1; - j += 1; + // move the next row for each of self's columns onto result. + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.push(self.group_key_cols[col_i][self_i]); + } + + // merge all the aggregates for this group key. + for (col_i, col) in result.aggregate_cols.iter_mut().enumerate() { + let self_value = self.aggregate_cols[col_i].value(self_i); + let other_value = other.aggregate_cols[col_i].value(other_i); + let (_, agg_type, _) = &self.schema.aggregate_columns[col_i]; + col.push(match agg_type { + AggregateType::Count => self_value + other_value, + AggregateType::Min => match self_value.partial_cmp(&other_value) { + Some(ord) => match ord { + Ordering::Less => self_value, + Ordering::Equal => self_value, + Ordering::Greater => other_value, + }, + None => self_value, + }, + AggregateType::Max => match self_value.partial_cmp(&other_value) { + Some(ord) => match ord { + Ordering::Less => other_value, + Ordering::Equal => other_value, + Ordering::Greater => self_value, + }, + None => self_value, + }, + AggregateType::Sum => self_value + other_value, + _ => unimplemented!("first/last not implemented"), + }); + } + self_i += 1; + other_i += 1; } Ordering::Greater => { - result.group_keys.push(other.group_keys[j].clone()); - result.aggregates.push(other.aggregates[j].clone()); - j += 1; + // move the next row for each of other's columns onto result. + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.push(other.group_key_cols[col_i][other_i]); + } + for (col_i, col) in result.aggregate_cols.iter_mut().enumerate() { + col.push(other.aggregate_cols[col_i].value(other_i)); + } + other_i += 1; } } } - result } - /// Executes a mutable sort of the rows in the result set based on the - /// lexicographic order of each group key column. - /// - /// TODO(edd): this has really poor performance. It clones the underlying - /// vectors rather than sorting them in place. - pub fn sort(&mut self) { - // The permutation crate lets you execute a sort on anything implements - // `Ord` and return the sort order, which can then be applied to other - // columns. - let perm = permutation::sort(self.group_keys.as_slice()); - self.group_keys = perm.apply_slice(self.group_keys.as_slice()); - self.aggregates = perm.apply_slice(self.aggregates.as_slice()); + // Executes a mutable sort of the results based on the lexicographic order + // of each group key columns. + // + // Given these group key columns: + // + // [foo [zoo [zoo + // foo bar zoo + // bar bar bar + // bar] bar] zoo] + // + // `sort` would result them becoming: + // + // [bar [bar [bar + // bar bar zoo + // foo bar zoo + // foo] zoo] zoo] + // + // The same permutation is also applied to the aggregate columns. + // + fn sort(&mut self) { + if self.group_keys_sorted { + return; + } + + // Create a vector of group keys, which allows us to determine a + // permutation by which we should sort all columns. + let mut group_keys = (0..self.rows()) + .map(|i| GroupKey::new(&self.group_key_cols, i)) + .collect::>(); + + // sort the vector of group keys, which will give us a permutation + // that we can apply to all of the columns. + group_keys.sort_unstable_by(|a, b| { + let cols = a.len(); + for i in 0..cols { + match a.columns[i][a.row_offset].partial_cmp(&b.columns[i][b.row_offset]) { + Some(ord) => { + if matches!(ord, Ordering::Equal) { + continue; + } + return ord; + } + None => continue, + } + } + + std::cmp::Ordering::Equal + }); + + // Now create a permutation by looking at how the row_offsets have been + // ordered in the `group_keys` array. + let perm = permutation::Permutation::from_vec( + group_keys + .iter() + .map(|gk| gk.row_offset) + .collect::>(), + ); + assert_eq!(perm.len(), self.rows()); + + // Apply that permutation to all of the columns. + for col in self.group_key_cols.iter_mut() { + *col = perm.apply_slice(col.as_slice()); + } + + for col in self.aggregate_cols.iter_mut() { + col.sort_with_permutation(&perm); + } + self.group_keys_sorted = true; } +} - pub fn add_row( - &mut self, - group_key: Vec>, - aggregates: Vec>, - ) { - self.group_keys.push(GroupKey(group_key)); - self.aggregates.push(AggregateResults(aggregates)); +// The `groupKey` struct is a wrapper over a specific row of data in grouping +// columns. +// +// Rather than pivot the columns into a row-wise orientation to sort them, we +// can effectively sort a projection across them (`row_offset`) storing +// `groupKey`s in a vector and sorting that. +struct GroupKey<'a> { + columns: &'a [Vec>], + row_offset: usize, +} + +impl<'a> GroupKey<'a> { + fn new(columns: &'a [Vec>], offset: usize) -> Self { + Self { + columns, + row_offset: offset, + } + } + + // The number of columns comprising the `GroupKey`. + fn len(&self) -> usize { + self.columns.len() } } impl TryFrom> for RecordBatch { type Error = Error; - fn try_from(result: ReadAggregateResult<'_>) -> Result { + fn try_from(mut result: ReadAggregateResult<'_>) -> Result { let schema = internal_types::schema::Schema::try_from(result.schema()) .map_err(|source| Error::SchemaError { source })?; let arrow_schema: arrow_deps::arrow::datatypes::SchemaRef = schema.into(); - // Build the columns for the group keys. This involves pivoting the - // row-wise group keys into column-wise data. - let mut group_column_builders = (0..result.schema.group_columns.len()) - .map(|_| { - arrow::array::StringBuilder::with_capacity( - result.cardinality(), - result.cardinality() * 8, // arbitrarily picked for now - ) - }) - .collect::>(); - - // build each column for a group key value row by row. - for gk in result.group_keys.iter() { - for (i, v) in gk.0.iter().enumerate() { - group_column_builders[i] - .append_value(v.string()) - .map_err(|source| Error::ArrowError { source })?; - } - } - // Add the group columns to the set of column data for the record batch. let mut columns: Vec> = Vec::with_capacity(result.schema.len()); - for col in group_column_builders.iter_mut() { - columns.push(Arc::new(col.finish())); + + for (_, data_type) in &result.schema.group_columns { + match data_type { + LogicalDataType::String => { + columns.push(Arc::new(array::StringArray::from( + result.group_key_cols.remove(0), // move column out of result + ))); + } + _ => panic!("only String currently supported as group column"), + } } - // For the aggregate columns, build one column at a time, repeatedly - // iterating rows until all columns have been built. - // - // TODO(edd): I don't like this *at all*. I'm going to refactor the way - // aggregates are produced. - for (i, (_, _, data_type)) in result.schema.aggregate_columns.iter().enumerate() { + for (_, _, data_type) in &result.schema.aggregate_columns { match data_type { LogicalDataType::Integer => { - let mut builder = array::Int64Builder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_i64_scalar()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::Int64Array::from( + result.aggregate_cols.remove(0).take_as_i64(), + ))); } LogicalDataType::Unsigned => { - let mut builder = array::UInt64Builder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_u64_scalar()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::UInt64Array::from( + result.aggregate_cols.remove(0).take_as_u64(), + ))); } LogicalDataType::Float => { - let mut builder = array::Float64Builder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_f64_scalar()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::Float64Array::from( + result.aggregate_cols.remove(0).take_as_f64(), + ))); } LogicalDataType::String => { - let mut builder = array::StringBuilder::new(result.cardinality()); - for agg_row in &result.aggregates { - match agg_row.0[i].try_as_str() { - Some(s) => builder.append_value(s).context(ArrowError)?, - None => builder.append_null().context(ArrowError)?, - } - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::StringArray::from( + result + .aggregate_cols + .remove(0) + .take_as_str() + .iter() + .map(|x| x.as_deref()) + .collect::>(), + ))); } LogicalDataType::Binary => { - let mut builder = array::BinaryBuilder::new(result.cardinality()); - for agg_row in &result.aggregates { - match agg_row.0[i].try_as_bytes() { - Some(s) => builder.append_value(s).context(ArrowError)?, - None => builder.append_null().context(ArrowError)?, - } - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::BinaryArray::from( + result + .aggregate_cols + .remove(0) + .take_as_bytes() + .iter() + .map(|x| x.as_deref()) + .collect::>(), + ))); } LogicalDataType::Boolean => { - let mut builder = array::BooleanBuilder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_bool()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::BooleanArray::from( + result.aggregate_cols.remove(0).take_as_bool(), + ))); } } } + // everything has been moved and copied into record batch. + assert!(result.group_key_cols.is_empty()); + assert!(result.aggregate_cols.is_empty()); + // try_new only returns an error if the schema is invalid or the number // of rows on columns differ. We have full control over both so there // should never be an error to return... @@ -1989,8 +2125,8 @@ impl TryFrom> for RecordBatch { impl PartialEq for ReadAggregateResult<'_> { fn eq(&self, other: &Self) -> bool { self.schema() == other.schema() - && self.group_keys == other.group_keys - && self.aggregates == other.aggregates + && self.group_key_cols == other.group_key_cols + && self.aggregate_cols == other.aggregate_cols } } @@ -2015,24 +2151,27 @@ impl std::fmt::Display for &ReadAggregateResult<'_> { } // There may or may not be group keys - let expected_rows = self.aggregates.len(); + let expected_rows = self.rows(); for row in 0..expected_rows { if row > 0 { writeln!(f)?; } // write row for group by columns - if !self.group_keys.is_empty() { - for value in self.group_keys[row].0.iter() { - write!(f, "{},", value)?; + if self.is_grouped_aggregate() { + for col in &self.group_key_cols { + match col[row] { + Some(v) => write!(f, "{},", v)?, + None => write!(f, "NULL,")?, + } } } // write row for aggregate columns - for (col_i, agg) in self.aggregates[row].0.iter().enumerate() { - write!(f, "{}", agg)?; - if col_i < self.aggregates[row].0.len() - 1 { - write!(f, ",")?; + for (i, col) in self.aggregate_cols.iter().enumerate() { + col.write_value(row, f)?; + if i < self.aggregate_cols.len() - 1 { + write!(f, ",")? } } } @@ -2235,7 +2374,7 @@ west,4 } #[test] - fn read_group() { + fn read_aggregate() { let mut columns = BTreeMap::new(); let tc = ColumnType::Time(Column::from(&[1_i64, 2, 3, 4, 5, 6][..])); columns.insert("time".to_string(), tc); @@ -2279,20 +2418,20 @@ west,4 // test queries with no predicates and grouping on low cardinality // columns - read_group_all_rows_all_rle(&row_group); + read_aggregate_all_rows_all_rle(&row_group); // test row group queries that group on fewer than five columns. - read_group_hash_u128_key(&row_group); + read_aggregate_hash_u128_key(&row_group); // test row group queries that use a vector-based group key. - read_group_hash_vec_key(&row_group); + read_aggregate_hash_vec_key(&row_group); // test row group queries that only group on one column. - read_group_single_groupby_column(&row_group); + read_aggregate_single_groupby_column(&row_group); } // the read_group path where grouping is on fewer than five columns. - fn read_group_hash_u128_key(row_group: &RowGroup) { + fn read_aggregate_hash_u128_key(row_group: &RowGroup) { let cases = vec![ ( Predicate::with_time_range(&[], 0, 7), // all time but without explicit pred @@ -2364,7 +2503,7 @@ west,prod,POST,4 // the read_group path where grouping is on five or more columns. This will // ensure that the `read_group_hash_with_vec_key` path is exercised. - fn read_group_hash_vec_key(row_group: &RowGroup) { + fn read_aggregate_hash_vec_key(row_group: &RowGroup) { let cases = vec![( Predicate::with_time_range(&[], 0, 7), // all time but with explicit pred vec!["region", "method", "env", "letters", "numbers"], @@ -2387,7 +2526,7 @@ west,POST,prod,Bravo,two,203 } // the read_group path where grouping is on a single column. - fn read_group_single_groupby_column(row_group: &RowGroup) { + fn read_aggregate_single_groupby_column(row_group: &RowGroup) { let cases = vec![( Predicate::with_time_range(&[], 0, 7), // all time but with explicit pred vec!["method"], @@ -2406,7 +2545,7 @@ PUT,203 } } - fn read_group_all_rows_all_rle(row_group: &RowGroup) { + fn read_aggregate_all_rows_all_rle(row_group: &RowGroup) { let cases = vec![ ( Predicate::default(), @@ -2467,7 +2606,7 @@ west,POST,304,101,203 } #[test] - fn row_group_could_satisfy_predicate() { + fn row_aggregate_could_satisfy_predicate() { let mut columns = BTreeMap::new(); let tc = ColumnType::Time(Column::from(&[1_i64, 2, 3, 4, 5, 6][..])); columns.insert("time".to_string(), tc); @@ -2523,7 +2662,7 @@ west,POST,304,101,203 } #[test] - fn row_group_satisfies_predicate() { + fn row_aggregate_satisfies_predicate() { let mut columns = BTreeMap::new(); let tc = ColumnType::Time(Column::from(&[1_i64, 2, 3, 4, 5, 6][..])); columns.insert("time".to_string(), tc); @@ -2612,7 +2751,7 @@ west,POST,304,101,203 } #[test] - fn read_group_result_display() { + fn read_aggregate_result_display() { let mut result = ReadAggregateResult { schema: ResultSchema { select_columns: vec![], @@ -2639,37 +2778,27 @@ west,POST,304,101,203 ), ], }, - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), - GroupKey(vec![Value::String("west"), Value::String("host-a")]), - GroupKey(vec![Value::String("west"), Value::String("host-c")]), - GroupKey(vec![Value::String("west"), Value::String("host-d")]), + group_key_cols: vec![ + vec![ + Some("east"), + Some("east"), + Some("west"), + Some("west"), + Some("west"), + ], + vec![ + Some("host-a"), + Some("host-b"), + Some("host-a"), + Some("host-c"), + Some("host-d"), + ], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(10)), - AggregateResult::Count(3), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(4), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(25)), - AggregateResult::Count(3), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(21)), - AggregateResult::Count(1), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(11)), - AggregateResult::Count(9), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20), Some(25), Some(21), Some(11)]), + AggregateVec::Count(vec![Some(3), Some(4), Some(3), Some(1), Some(9)]), ], group_keys_sorted: false, - ..Default::default() }; // Debug implementation @@ -2697,7 +2826,7 @@ west,host-d,11,9 // results don't have to have group keys. result.schema.group_columns = vec![]; - result.group_keys = vec![]; + result.group_key_cols = vec![]; // Debug implementation assert_eq!( @@ -2724,7 +2853,7 @@ west,host-d,11,9 } #[test] - fn read_group_result_merge() { + fn read_aggregate_result_merge() { let schema = ResultSchema { group_columns: vec![ ( @@ -2756,24 +2885,18 @@ west,host-d,11,9 ..Default::default() }; - let mut other_result = ReadAggregateResult { + let other_result = ReadAggregateResult { schema: schema.clone(), + group_key_cols: vec![ + vec![Some("east"), Some("east")], + vec![Some("host-a"), Some("host-b")], + ], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20)]), + AggregateVec::Count(vec![Some(3), Some(4)]), + ], ..Default::default() }; - other_result.add_row( - vec![Value::String("east"), Value::String("host-a")], - vec![ - AggregateResult::Sum(Scalar::I64(10)), - AggregateResult::Count(3), - ], - ); - other_result.add_row( - vec![Value::String("east"), Value::String("host-b")], - vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(4), - ], - ); // merging something into nothing results in having a copy of something. result = result.merge(other_result.clone()); @@ -2786,19 +2909,13 @@ west,host-d,11,9 result, ReadAggregateResult { schema: schema.clone(), - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), + group_key_cols: vec![ + vec![Some("east"), Some("east")], + vec![Some("host-a"), Some("host-b")], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(6), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(40)), - AggregateResult::Count(8), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(20), Some(40)]), + AggregateVec::Count(vec![Some(6), Some(8)]), ], ..Default::default() } @@ -2806,41 +2923,28 @@ west,host-d,11,9 // merging a result in with different group keys merges those group // keys in. - let mut other_result = ReadAggregateResult { + let other_result = ReadAggregateResult { schema: schema.clone(), + group_key_cols: vec![vec![Some("north")], vec![Some("host-a")]], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(-5)]), + AggregateVec::Count(vec![Some(2)]), + ], ..Default::default() }; - other_result.add_row( - vec![Value::String("north"), Value::String("host-a")], - vec![ - AggregateResult::Sum(Scalar::I64(-5)), - AggregateResult::Count(2), - ], - ); result = result.merge(other_result.clone()); assert_eq!( result, ReadAggregateResult { schema: schema.clone(), - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), - GroupKey(vec![Value::String("north"), Value::String("host-a")]), + group_key_cols: vec![ + vec![Some("east"), Some("east"), Some("north")], + vec![Some("host-a"), Some("host-b"), Some("host-a")], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(6), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(40)), - AggregateResult::Count(8), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(-5)), - AggregateResult::Count(2), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(20), Some(40), Some(-5)]), + AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), ], ..Default::default() } @@ -2857,30 +2961,45 @@ west,host-d,11,9 result, ReadAggregateResult { schema, - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), - GroupKey(vec![Value::String("north"), Value::String("host-a")]), + group_key_cols: vec![ + vec![Some("east"), Some("east"), Some("north")], + vec![Some("host-a"), Some("host-b"), Some("host-a")], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(6), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(40)), - AggregateResult::Count(8), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(-5)), - AggregateResult::Count(2), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(20), Some(40), Some(-5)]), + AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), ], ..Default::default() } ); } + #[test] + fn read_aggregate_result_sort() { + let mut result = ReadAggregateResult { + schema: ResultSchema::default(), + group_key_cols: vec![ + vec![Some("west"), Some("east"), Some("north")], + vec![Some("host-c"), Some("host-c"), Some("host-c")], + vec![Some("pro"), Some("stag"), Some("dev")], + ], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20), Some(-5)]), + AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), + ], + ..Default::default() + }; + result.sort(); + + assert_eq!( + format!("{}", &result), + "east,host-c,stag,20,8 +north,host-c,dev,-5,2 +west,host-c,pro,10,6 +" + ); + } + #[test] fn column_meta_equal() { let col1 = ColumnMeta { diff --git a/read_buffer/src/schema.rs b/read_buffer/src/schema.rs index 9d9828ff2a..a9964356b1 100644 --- a/read_buffer/src/schema.rs +++ b/read_buffer/src/schema.rs @@ -50,6 +50,7 @@ impl ResultSchema { self.len() == 0 } + /// The total number of columns the schema represents. pub fn len(&self) -> usize { self.select_columns.len() + self.group_columns.len() + self.aggregate_columns.len() } diff --git a/read_buffer/src/table.rs b/read_buffer/src/table.rs index f11c56e1b6..23f630cf81 100644 --- a/read_buffer/src/table.rs +++ b/read_buffer/src/table.rs @@ -10,7 +10,7 @@ use arrow_deps::arrow::record_batch::RecordBatch; use internal_types::selection::Selection; use snafu::{ensure, Snafu}; -use crate::row_group::{self, ColumnName, GroupKey, Predicate, RowGroup}; +use crate::row_group::{self, ColumnName, Predicate, RowGroup}; use crate::schema::{AggregateType, ColumnType, LogicalDataType, ResultSchema}; use crate::value::{AggregateResult, Scalar, Value}; #[derive(Debug, Snafu)] @@ -273,7 +273,7 @@ impl Table { _group_columns: Vec>, _aggregates: Vec<(ColumnName<'a>, AggregateType)>, _window: i64, - ) -> BTreeMap, Vec<(ColumnName<'a>, AggregateResult<'_>)>> { + ) -> BTreeMap, Vec<(ColumnName<'a>, AggregateResult<'_>)>> { // identify segments where time range and predicates match could match // using segment meta data, and then execute against those segments and // merge results. @@ -942,7 +942,7 @@ mod test { use crate::row_group::{BinaryExpr, ColumnType, ReadAggregateResult}; use crate::schema; use crate::schema::LogicalDataType; - use crate::value::{OwnedValue, Scalar}; + use crate::value::{AggregateVec, OwnedValue, Scalar}; #[test] fn meta_data_update_with() { @@ -1237,7 +1237,7 @@ mod test { #[test] fn read_aggregate_result_display() { - let mut result_a = ReadAggregateResult { + let result_a = ReadAggregateResult { schema: ResultSchema { select_columns: vec![], group_columns: vec![ @@ -1256,14 +1256,12 @@ mod test { LogicalDataType::Integer, )], }, + group_key_cols: vec![vec![Some("east")], vec![Some("host-a")]], + aggregate_cols: vec![AggregateVec::SumI64(vec![Some(10)])], ..ReadAggregateResult::default() }; - result_a.add_row( - vec![Value::String("east"), Value::String("host-a")], - vec![AggregateResult::Sum(Scalar::I64(10))], - ); - let mut result_b = ReadAggregateResult { + let result_b = ReadAggregateResult { schema: ResultSchema { select_columns: vec![], group_columns: vec![ @@ -1282,12 +1280,10 @@ mod test { LogicalDataType::Integer, )], }, + group_key_cols: vec![vec![Some("west")], vec![Some("host-b")]], + aggregate_cols: vec![AggregateVec::SumI64(vec![Some(100)])], ..Default::default() }; - result_b.add_row( - vec![Value::String("west"), Value::String("host-b")], - vec![AggregateResult::Sum(Scalar::I64(100))], - ); let results = DisplayReadAggregateResults(vec![result_a, result_b]); //Display implementation assert_eq!( diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index 94ecdd8cb1..85ec979c5d 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -1,13 +1,13 @@ -use std::{collections::BTreeSet, convert::TryFrom}; +use std::{collections::BTreeSet, convert::TryFrom, fmt::Formatter}; use std::{mem::size_of, sync::Arc}; use arrow_deps::arrow; use crate::{AggregateType, LogicalDataType}; -#[derive(Clone)] -pub enum AggregateVec<'a> { - Count(Vec), +#[derive(Clone, PartialEq, Debug)] +pub enum AggregateVec { + Count(Vec>), SumI64(Vec>), SumU64(Vec>), @@ -16,45 +16,104 @@ pub enum AggregateVec<'a> { MinU64(Vec>), MinI64(Vec>), MinF64(Vec>), - MinString(Vec>), - MinBytes(Vec>), + MinString(Vec>), + MinBytes(Vec>>), MinBool(Vec>), MaxU64(Vec>), MaxI64(Vec>), MaxF64(Vec>), - MaxString(Vec>), - MaxBytes(Vec>), + MaxString(Vec>), + MaxBytes(Vec>>), MaxBool(Vec>), FirstU64((Vec>, Vec>)), FirstI64((Vec>, Vec>)), FirstF64((Vec>, Vec>)), - FirstString((Vec>, Vec>)), - FirstBytes((Vec>, Vec>)), + FirstString((Vec>, Vec>)), + FirstBytes((Vec>>, Vec>)), FirstBool((Vec>, Vec>)), LastU64((Vec>, Vec>)), LastI64((Vec>, Vec>)), LastF64((Vec>, Vec>)), - LastString((Vec>, Vec>)), - LastBytes((Vec>, Vec>)), + LastString((Vec>, Vec>)), + LastBytes((Vec>>, Vec>)), LastBool((Vec>, Vec>)), } -impl AggregateVec<'_> { +impl AggregateVec { + pub fn len(&self) -> usize { + match self { + Self::Count(arr) => arr.len(), + Self::SumI64(arr) => arr.len(), + Self::SumU64(arr) => arr.len(), + Self::SumF64(arr) => arr.len(), + Self::MinU64(arr) => arr.len(), + Self::MinI64(arr) => arr.len(), + Self::MinF64(arr) => arr.len(), + Self::MinString(arr) => arr.len(), + Self::MinBytes(arr) => arr.len(), + Self::MinBool(arr) => arr.len(), + Self::MaxU64(arr) => arr.len(), + Self::MaxI64(arr) => arr.len(), + Self::MaxF64(arr) => arr.len(), + Self::MaxString(arr) => arr.len(), + Self::MaxBytes(arr) => arr.len(), + Self::MaxBool(arr) => arr.len(), + Self::FirstU64((arr, _)) => arr.len(), + Self::FirstI64((arr, _)) => arr.len(), + Self::FirstF64((arr, _)) => arr.len(), + Self::FirstString((arr, _)) => arr.len(), + Self::FirstBytes((arr, _)) => arr.len(), + Self::FirstBool((arr, _)) => arr.len(), + Self::LastU64((arr, _)) => arr.len(), + Self::LastI64((arr, _)) => arr.len(), + Self::LastF64((arr, _)) => arr.len(), + Self::LastString((arr, _)) => arr.len(), + Self::LastBytes((arr, _)) => arr.len(), + Self::LastBool((arr, _)) => arr.len(), + } + } + + /// Returns the value specified by `offset`. + pub fn value(&self, offset: usize) -> Value<'_> { + match &self { + Self::Count(arr) => Value::from(arr[offset]), + Self::SumI64(arr) => Value::from(arr[offset]), + Self::SumU64(arr) => Value::from(arr[offset]), + Self::SumF64(arr) => Value::from(arr[offset]), + Self::MinU64(arr) => Value::from(arr[offset]), + Self::MinI64(arr) => Value::from(arr[offset]), + Self::MinF64(arr) => Value::from(arr[offset]), + Self::MinString(arr) => Value::from(arr[offset].as_deref()), + Self::MinBytes(arr) => Value::from(arr[offset].as_deref()), + Self::MinBool(arr) => Value::from(arr[offset]), + Self::MaxU64(arr) => Value::from(arr[offset]), + Self::MaxI64(arr) => Value::from(arr[offset]), + Self::MaxF64(arr) => Value::from(arr[offset]), + Self::MaxString(arr) => Value::from(arr[offset].as_deref()), + Self::MaxBytes(arr) => Value::from(arr[offset].as_deref()), + Self::MaxBool(arr) => Value::from(arr[offset]), + _ => unimplemented!("first/last not yet implemented"), + } + } + pub fn update(&mut self, values: &Values<'_>, row_id: usize, offset: usize) { match self { - AggregateVec::Count(arr) => { + Self::Count(arr) => { if values.is_null(row_id) { return; } else if offset >= arr.len() { - arr.resize(offset + 1, 0); + arr.resize(offset + 1, None); } - arr[offset] += 1; + match &mut arr[offset] { + Some(v) => *v += 1, + None => arr[offset] = Some(1), + } } - AggregateVec::SumI64(arr) => { + Self::SumI64(arr) => { if values.is_null(row_id) { return; } else if offset >= arr.len() { @@ -66,76 +125,871 @@ impl AggregateVec<'_> { None => arr[offset] = Some(values.value_i64(row_id)), } } - AggregateVec::SumU64(_) => {} - AggregateVec::SumF64(_) => {} - AggregateVec::MinU64(_) => {} - AggregateVec::MinI64(_) => {} - AggregateVec::MinF64(_) => {} - AggregateVec::MinString(_) => {} - AggregateVec::MinBytes(_) => {} - AggregateVec::MinBool(_) => {} - AggregateVec::MaxU64(_) => {} - AggregateVec::MaxI64(_) => {} - AggregateVec::MaxF64(_) => {} - AggregateVec::MaxString(_) => {} - AggregateVec::MaxBytes(_) => {} - AggregateVec::MaxBool(_) => {} - AggregateVec::FirstU64(_) => {} - AggregateVec::FirstI64(_) => {} - AggregateVec::FirstF64(_) => {} - AggregateVec::FirstString(_) => {} - AggregateVec::FirstBytes(_) => {} - AggregateVec::FirstBool(_) => {} - AggregateVec::LastU64(_) => {} - AggregateVec::LastI64(_) => {} - AggregateVec::LastF64(_) => {} - AggregateVec::LastString(_) => {} - AggregateVec::LastBytes(_) => {} - AggregateVec::LastBool(_) => {} + Self::SumU64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v += values.value_u64(row_id), + None => arr[offset] = Some(values.value_u64(row_id)), + } + } + Self::SumF64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v += values.value_f64(row_id), + None => arr[offset] = Some(values.value_f64(row_id)), + } + } + Self::MinU64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_u64(row_id)), + None => arr[offset] = Some(values.value_u64(row_id)), + } + } + Self::MinI64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_i64(row_id)), + None => arr[offset] = Some(values.value_i64(row_id)), + } + } + Self::MinF64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_f64(row_id)), + None => arr[offset] = Some(values.value_f64(row_id)), + } + } + Self::MinString(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_str(row_id); + if other < v.as_str() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_str(row_id).to_owned()), + } + } + Self::MinBytes(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_bytes(row_id); + if other < v.as_slice() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_bytes(row_id).to_owned()), + } + } + Self::MinBool(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_bool(row_id)), + None => arr[offset] = Some(values.value_bool(row_id)), + } + } + Self::MaxU64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_u64(row_id)), + None => arr[offset] = Some(values.value_u64(row_id)), + } + } + Self::MaxI64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_i64(row_id)), + None => arr[offset] = Some(values.value_i64(row_id)), + } + } + Self::MaxF64(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_f64(row_id)), + None => arr[offset] = Some(values.value_f64(row_id)), + } + } + Self::MaxString(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_str(row_id); + if other > v.as_str() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_str(row_id).to_owned()), + } + } + Self::MaxBytes(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_bytes(row_id); + if other > v.as_slice() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_bytes(row_id).to_owned()), + } + } + Self::MaxBool(arr) => { + if values.is_null(row_id) { + return; + } else if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_bool(row_id)), + None => arr[offset] = Some(values.value_bool(row_id)), + } + } + // TODO - implement first/last + _ => unimplemented!("aggregate update not implemented"), } } /// Appends the provided value to the end of the aggregate vector. /// Panics if the type of `Value` does not satisfy the aggregate type. - pub fn push(&mut self, value: &Value<'_>) { + /// + /// Note: updating pushed first/last variants is not currently a supported + /// operation. + pub fn push(&mut self, value: Value<'_>) { match self { - AggregateVec::Count(arr) => arr.push(value.u64()), - AggregateVec::SumI64(arr) => arr.push(Some(value.i64())), - AggregateVec::SumU64(_) => {} - AggregateVec::SumF64(_) => {} - AggregateVec::MinU64(_) => {} - AggregateVec::MinI64(_) => {} - AggregateVec::MinF64(_) => {} - AggregateVec::MinString(_) => {} - AggregateVec::MinBytes(_) => {} - AggregateVec::MinBool(_) => {} - AggregateVec::MaxU64(_) => {} - AggregateVec::MaxI64(_) => {} - AggregateVec::MaxF64(_) => {} - AggregateVec::MaxString(_) => {} - AggregateVec::MaxBytes(_) => {} - AggregateVec::MaxBool(_) => {} - AggregateVec::FirstU64(_) => {} - AggregateVec::FirstI64(_) => {} - AggregateVec::FirstF64(_) => {} - AggregateVec::FirstString(_) => {} - AggregateVec::FirstBytes(_) => {} - AggregateVec::FirstBool(_) => {} - AggregateVec::LastU64(_) => {} - AggregateVec::LastI64(_) => {} - AggregateVec::LastF64(_) => {} - AggregateVec::LastString(_) => {} - AggregateVec::LastBytes(_) => {} - AggregateVec::LastBool(_) => {} + Self::Count(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::SumI64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::SumU64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::SumF64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::MinU64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::MinI64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::MinF64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::MinString(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::MinBytes(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::MinBool(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } + Self::MaxU64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::MaxI64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::MaxF64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::MaxString(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::MaxBytes(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::MaxBool(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } + Self::FirstU64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::FirstI64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::FirstF64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::FirstString((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::FirstBytes((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::FirstBool((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } + Self::LastU64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::LastI64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::LastF64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::LastString((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::LastBytes((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::LastBool((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } + } + } + + pub fn write_value(&self, offset: usize, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Count(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::SumI64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::SumU64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::SumF64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinU64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinI64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinF64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinString(arr) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinBytes(arr) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinBool(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxU64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxI64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxF64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxString(arr) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxBytes(arr) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxBool(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstU64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstI64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstF64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstString((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstBytes((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstBool((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastU64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastI64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastF64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastString((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastBytes((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastBool((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + } + Ok(()) + } + + // Consumes self and returns the inner `Vec>`. + pub fn as_i64(&self) -> &Vec> { + match self { + Self::SumI64(arr) => arr, + Self::MinI64(arr) => arr, + Self::MaxI64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn as_u64(&self) -> &Vec> { + match self { + Self::Count(arr) => arr, + Self::SumU64(arr) => arr, + Self::MinU64(arr) => arr, + Self::MaxU64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn as_f64(&self) -> &Vec> { + match self { + Self::SumF64(arr) => arr, + Self::MinF64(arr) => arr, + Self::MaxF64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn as_str(&self) -> &Vec> { + match self { + Self::MinString(arr) => arr, + Self::MaxString(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn as_bytes(&self) -> &Vec>> { + match self { + Self::MinBytes(arr) => arr, + Self::MaxBytes(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn as_bool(&self) -> &Vec> { + match self { + Self::MinBool(arr) => arr, + Self::MaxBool(arr) => arr, + _ => panic!("cannot convert {} to Vec", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn take_as_i64(self) -> Vec> { + match self { + Self::SumI64(arr) => arr, + Self::MinI64(arr) => arr, + Self::MaxI64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn take_as_u64(self) -> Vec> { + match self { + Self::Count(arr) => arr, + Self::SumU64(arr) => arr, + Self::MinU64(arr) => arr, + Self::MaxU64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn take_as_f64(self) -> Vec> { + match self { + Self::SumF64(arr) => arr, + Self::MinF64(arr) => arr, + Self::MaxF64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn take_as_str(self) -> Vec> { + match self { + Self::MinString(arr) => arr, + Self::MaxString(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>>`. + pub fn take_as_bytes(self) -> Vec>> { + match self { + Self::MinBytes(arr) => arr, + Self::MaxBytes(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn take_as_bool(self) -> Vec> { + match self { + Self::MinBool(arr) => arr, + Self::MaxBool(arr) => arr, + _ => panic!("cannot convert {} to Vec", self), + } + } + + /// Appends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_i64(&mut self, itr: impl Iterator>) { + match self { + Self::SumI64(arr) => { + arr.extend(itr); + } + Self::MinI64(arr) => { + arr.extend(itr); + } + Self::MaxI64(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Appends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_u64(&mut self, itr: impl Iterator>) { + match self { + Self::Count(arr) => { + arr.extend(itr); + } + Self::SumU64(arr) => { + arr.extend(itr); + } + Self::MinU64(arr) => { + arr.extend(itr); + } + Self::MaxU64(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Appends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_f64(&mut self, itr: impl Iterator>) { + match self { + Self::SumF64(arr) => { + arr.extend(itr); + } + Self::MinF64(arr) => { + arr.extend(itr); + } + Self::MaxF64(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Appends the `AggregateVec` with the provided `Option<&str>` iterator. + pub fn extend_with_str(&mut self, itr: impl Iterator>) { + match self { + Self::MinString(arr) => { + arr.extend(itr); + } + Self::MaxString(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Appends the `AggregateVec` with the provided `Option>` iterator. + pub fn extend_with_bytes(&mut self, itr: impl Iterator>>) { + match self { + Self::MinBytes(arr) => { + arr.extend(itr); + } + Self::MaxBytes(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Appends the `AggregateVec` with the provided `Option<&[u8]>` iterator. + pub fn extend_with_bool(&mut self, itr: impl Iterator>) { + match self { + Self::MinBool(arr) => { + arr.extend(itr); + } + Self::MaxBool(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + pub fn sort_with_permutation(&mut self, p: &permutation::Permutation) { + match self { + Self::Count(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::SumI64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::SumU64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::SumF64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinU64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinI64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinF64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinString(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinBytes(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinBool(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxU64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxI64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxF64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxString(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxBytes(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxBool(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::FirstU64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstI64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstF64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstString((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstBytes((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstBool((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastU64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastI64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastF64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastString((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastBytes((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastBool((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } } } } -impl From<(&AggregateType, &LogicalDataType, usize)> for AggregateVec<'_> { +impl std::fmt::Display for AggregateVec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Count(_) => write!(f, "Count"), + Self::SumI64(_) => write!(f, "Sum"), + Self::SumU64(_) => write!(f, "Sum"), + Self::SumF64(_) => write!(f, "Sum"), + Self::MinU64(_) => write!(f, "Min"), + Self::MinI64(_) => write!(f, "Min"), + Self::MinF64(_) => write!(f, "Min"), + Self::MinString(_) => write!(f, "Min"), + Self::MinBytes(_) => write!(f, "Min>"), + Self::MinBool(_) => write!(f, "Min"), + Self::MaxU64(_) => write!(f, "Max"), + Self::MaxI64(_) => write!(f, "Max"), + Self::MaxF64(_) => write!(f, "Max"), + Self::MaxString(_) => write!(f, "Max"), + Self::MaxBytes(_) => write!(f, "Max>"), + Self::MaxBool(_) => write!(f, "Max"), + Self::FirstU64(_) => write!(f, "First"), + Self::FirstI64(_) => write!(f, "First"), + Self::FirstF64(_) => write!(f, "First"), + Self::FirstString(_) => write!(f, "First"), + Self::FirstBytes(_) => write!(f, "First>"), + Self::FirstBool(_) => write!(f, "First"), + Self::LastU64(_) => write!(f, "Last"), + Self::LastI64(_) => write!(f, "Last"), + Self::LastF64(_) => write!(f, "Last"), + Self::LastString(_) => write!(f, "Last"), + Self::LastBytes(_) => write!(f, "Last>"), + Self::LastBool(_) => write!(f, "Last"), + } + } +} + +impl From<(&AggregateType, &LogicalDataType, usize)> for AggregateVec { fn from(v: (&AggregateType, &LogicalDataType, usize)) -> Self { let length = v.2; match (v.0, v.1) { - (AggregateType::Count, _) => Self::Count(vec![0; length]), + (AggregateType::Count, _) => Self::Count(vec![None; length]), (AggregateType::First, LogicalDataType::Integer) => { Self::FirstI64((vec![None; length], vec![None; length])) } @@ -644,6 +1498,26 @@ impl<'a> std::ops::AddAssign<&Scalar> for &mut Scalar { } } +impl std::ops::Add for Scalar { + type Output = Self; + + fn add(self, other: Self) -> Self { + match (self, other) { + (Self::Null, Self::Null) => Self::Null, + (Self::Null, Self::I64(_)) => other, + (Self::Null, Self::U64(_)) => other, + (Self::Null, Self::F64(_)) => other, + (Self::I64(_), Self::Null) => self, + (Self::I64(a), Self::I64(b)) => Self::I64(a + b), + (Self::U64(_), Self::Null) => self, + (Self::U64(a), Self::U64(b)) => Self::U64(a + b), + (Self::F64(_), Self::Null) => self, + (Self::F64(a), Self::F64(b)) => Self::F64(a + b), + (a, b) => panic!("{:?} + {:?} is an unsupported operation", a, b), + } + } +} + impl std::fmt::Display for Scalar { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -728,7 +1602,7 @@ pub enum Value<'a> { Scalar(Scalar), } -impl Value<'_> { +impl<'a> Value<'a> { pub fn is_null(&self) -> bool { matches!(self, Self::Null) } @@ -740,30 +1614,44 @@ impl Value<'_> { panic!("cannot unwrap Value to Scalar"); } - pub fn i64(&self) -> i64 { + pub fn i64(self) -> i64 { if let Self::Scalar(Scalar::I64(v)) = self { - return *v; + return v; } panic!("cannot unwrap Value to i64"); } - pub fn u64(&self) -> u64 { + pub fn u64(self) -> u64 { if let Self::Scalar(Scalar::U64(v)) = self { - return *v; + return v; } panic!("cannot unwrap Value to u64"); } - pub fn string(&self) -> &str { + pub fn f64(self) -> f64 { + if let Self::Scalar(Scalar::F64(v)) = self { + return v; + } + panic!("cannot unwrap Value to u64"); + } + + pub fn str(self) -> &'a str { if let Self::String(s) = self { return s; } panic!("cannot unwrap Value to String"); } - pub fn bool(&self) -> bool { + pub fn bytes(self) -> &'a [u8] { + if let Self::ByteArray(s) = self { + return s; + } + panic!("cannot unwrap Value to byte array"); + } + + pub fn bool(self) -> bool { if let Self::Boolean(b) = self { - return *b; + return b; } panic!("cannot unwrap Value to Scalar"); } @@ -792,6 +1680,45 @@ impl<'a> From<&'a str> for Value<'a> { } } +impl<'a> From> for Value<'a> { + fn from(v: Option<&'a str>) -> Self { + match v { + Some(s) => Self::String(s), + None => Self::Null, + } + } +} + +impl<'a> From<&'a [u8]> for Value<'a> { + fn from(v: &'a [u8]) -> Self { + Self::ByteArray(v) + } +} + +impl<'a> From> for Value<'a> { + fn from(v: Option<&'a [u8]>) -> Self { + match v { + Some(s) => Self::ByteArray(s), + None => Self::Null, + } + } +} + +impl<'a> From for Value<'a> { + fn from(v: bool) -> Self { + Self::Boolean(v) + } +} + +impl<'a> From> for Value<'a> { + fn from(v: Option) -> Self { + match v { + Some(s) => Self::Boolean(s), + None => Self::Null, + } + } +} + // Implementations of From trait for various concrete types. macro_rules! scalar_from_impls { ($(($variant:ident, $type:ident),)*) => { @@ -820,6 +1747,17 @@ scalar_from_impls! { (F64, f64), } +impl std::ops::Add for Value<'_> { + type Output = Self; + + fn add(self, other: Self) -> Self { + match (self, other) { + (Self::Scalar(a), Self::Scalar(b)) => Self::Scalar(a + b), + _ => panic!("unsupported operation on Value"), + } + } +} + /// Each variant is a typed vector of materialised values for a column. #[derive(Debug, PartialEq)] pub enum Values<'a> { @@ -914,6 +1852,48 @@ impl<'a> Values<'a> { _ => panic!("value cannot be returned as i64"), } } + + // Returns a value as an u64. Panics if not possible. + fn value_u64(&self, i: usize) -> u64 { + match &self { + Values::U64(c) => c[i], + Values::U64N(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as u64"), + } + } + + // Returns a value as an f64. Panics if not possible. + fn value_f64(&self, i: usize) -> f64 { + match &self { + Values::F64(c) => c[i], + Values::F64N(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as f64"), + } + } + + // Returns a value as a string. Panics if not possible. + fn value_str(&self, i: usize) -> &'a str { + match &self { + Values::String(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as &str"), + } + } + + // Returns a value as a binary array. Panics if not possible. + fn value_bytes(&self, i: usize) -> &'a [u8] { + match &self { + Values::ByteArray(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as &str"), + } + } + + // Returns a value as a bool. Panics if not possible. + fn value_bool(&self, i: usize) -> bool { + match &self { + Values::Bool(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as &str"), + } + } } /// Moves ownership of Values into an arrow `ArrayRef`. From ffa06e4f0e00c0d677845c04968841ab82a90c14 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Fri, 19 Mar 2021 15:41:20 +0000 Subject: [PATCH 082/104] test: more test coverage --- read_buffer/src/row_group.rs | 162 +++++++++----- read_buffer/src/value.rs | 404 ++++++++++++++++++++--------------- 2 files changed, 343 insertions(+), 223 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index db4bdf7b11..6c3231c2dc 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -545,7 +545,7 @@ impl RowGroup { .schema .aggregate_columns .iter() - .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) .collect::>(); // Maps each group key to an ordinal offset on output columns. This @@ -655,7 +655,7 @@ impl RowGroup { .schema .aggregate_columns .iter() - .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) .collect::>(); // Maps each group key to an ordinal offset on output columns. This @@ -779,7 +779,7 @@ impl RowGroup { .schema .aggregate_columns .iter() - .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) .collect::>(); let mut output_rows = 0; @@ -938,7 +938,7 @@ impl RowGroup { .schema .aggregate_columns .iter() - .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) .collect::>(); for (i, (col, agg_type)) in input_aggregate_columns.iter().enumerate() { @@ -1776,7 +1776,7 @@ impl<'row_group> ReadAggregateResult<'row_group> { .schema .aggregate_columns .iter() - .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type, 0))) + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) .collect::>(); let mut self_i = 0; @@ -1798,28 +1798,34 @@ impl<'row_group> ReadAggregateResult<'row_group> { { match data_type { LogicalDataType::Integer => { - let itr = other.aggregate_cols[col_i].as_i64().iter().cloned(); - result.aggregate_cols[col_i].extend_with_i64(itr); + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_i64(arr.take_as_i64().into_iter()); } LogicalDataType::Unsigned => { - let itr = other.aggregate_cols[col_i].as_u64().iter().cloned(); - result.aggregate_cols[col_i].extend_with_u64(itr); + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_u64(arr.take_as_u64().into_iter()); } LogicalDataType::Float => { - let itr = other.aggregate_cols[col_i].as_f64().iter().cloned(); - result.aggregate_cols[col_i].extend_with_f64(itr); + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_f64(arr.take_as_f64().into_iter()); } LogicalDataType::String => { - let itr = other.aggregate_cols[col_i].as_str().iter().cloned(); - result.aggregate_cols[col_i].extend_with_str(itr); + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_str(arr.take_as_str().into_iter()); } LogicalDataType::Binary => { - let itr = other.aggregate_cols[col_i].as_bytes().iter().cloned(); - result.aggregate_cols[col_i].extend_with_bytes(itr); + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bytes(arr.take_as_bytes().into_iter()); } LogicalDataType::Boolean => { - let itr = other.aggregate_cols[col_i].as_bool().iter().cloned(); - result.aggregate_cols[col_i].extend_with_bool(itr); + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bool(arr.take_as_bool().into_iter()); } } } @@ -1836,28 +1842,34 @@ impl<'row_group> ReadAggregateResult<'row_group> { { match data_type { LogicalDataType::Integer => { - let itr = self.aggregate_cols[col_i].as_i64().iter().cloned(); - result.aggregate_cols[col_i].extend_with_i64(itr); + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_i64(arr.take_as_i64().into_iter()); } LogicalDataType::Unsigned => { - let itr = self.aggregate_cols[col_i].as_u64().iter().cloned(); - result.aggregate_cols[col_i].extend_with_u64(itr); + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_u64(arr.take_as_u64().into_iter()); } LogicalDataType::Float => { - let itr = self.aggregate_cols[col_i].as_f64().iter().cloned(); - result.aggregate_cols[col_i].extend_with_f64(itr); + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_f64(arr.take_as_f64().into_iter()); } LogicalDataType::String => { - let itr = self.aggregate_cols[col_i].as_str().iter().cloned(); - result.aggregate_cols[col_i].extend_with_str(itr); + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_str(arr.take_as_str().into_iter()); } LogicalDataType::Binary => { - let itr = self.aggregate_cols[col_i].as_bytes().iter().cloned(); - result.aggregate_cols[col_i].extend_with_bytes(itr); + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bytes(arr.take_as_bytes().into_iter()); } LogicalDataType::Boolean => { - let itr = self.aggregate_cols[col_i].as_bool().iter().cloned(); - result.aggregate_cols[col_i].extend_with_bool(itr); + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bool(arr.take_as_bool().into_iter()); } } } @@ -1962,7 +1974,7 @@ impl<'row_group> ReadAggregateResult<'row_group> { // // The same permutation is also applied to the aggregate columns. // - fn sort(&mut self) { + pub fn sort(&mut self) { if self.group_keys_sorted { return; } @@ -2852,6 +2864,70 @@ west,host-d,11,9 ); } + #[test] + fn read_aggregate_result_sort() { + let mut result = ReadAggregateResult { + schema: ResultSchema::default(), // schema not needed for sorting. + group_key_cols: vec![ + vec![ + Some("east"), + Some("west"), + Some("west"), + Some("east"), + Some("west"), + ], + vec![ + Some("host-a"), + Some("host-c"), + Some("host-a"), + Some("host-d"), + Some("host-b"), + ], + ], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20), Some(25), Some(21), Some(11)]), + AggregateVec::Count(vec![Some(3), Some(4), Some(3), Some(1), Some(9)]), + ], + group_keys_sorted: false, + }; + + result.sort(); + + // Debug implementation + assert_eq!( + format!("{}", &result), + "east,host-a,10,3 +east,host-d,21,1 +west,host-a,25,3 +west,host-b,11,9 +west,host-c,20,4 +" + ); + + let mut result = ReadAggregateResult { + schema: ResultSchema::default(), + group_key_cols: vec![ + vec![Some("west"), Some("east"), Some("north")], + vec![Some("host-c"), Some("host-c"), Some("host-c")], + vec![Some("pro"), Some("stag"), Some("dev")], + ], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20), Some(-5)]), + AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), + ], + ..Default::default() + }; + result.sort(); + + assert_eq!( + format!("{}", &result), + "east,host-c,stag,20,8 +north,host-c,dev,-5,2 +west,host-c,pro,10,6 +" + ); + } + #[test] fn read_aggregate_result_merge() { let schema = ResultSchema { @@ -2974,32 +3050,6 @@ west,host-d,11,9 ); } - #[test] - fn read_aggregate_result_sort() { - let mut result = ReadAggregateResult { - schema: ResultSchema::default(), - group_key_cols: vec![ - vec![Some("west"), Some("east"), Some("north")], - vec![Some("host-c"), Some("host-c"), Some("host-c")], - vec![Some("pro"), Some("stag"), Some("dev")], - ], - aggregate_cols: vec![ - AggregateVec::SumI64(vec![Some(10), Some(20), Some(-5)]), - AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), - ], - ..Default::default() - }; - result.sort(); - - assert_eq!( - format!("{}", &result), - "east,host-c,stag,20,8 -north,host-c,dev,-5,2 -west,host-c,pro,10,6 -" - ); - } - #[test] fn column_meta_equal() { let col1 = ColumnMeta { diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index 85ec979c5d..2092c0d5da 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -99,12 +99,18 @@ impl AggregateVec { } } + /// Updates with a new value located in the provided input column help in + /// `Values`. + /// + /// Panics if the type of `Value` does not satisfy the aggregate type. pub fn update(&mut self, values: &Values<'_>, row_id: usize, offset: usize) { + if values.is_null(row_id) { + return; + } + match self { Self::Count(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -114,9 +120,7 @@ impl AggregateVec { } } Self::SumI64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -126,9 +130,7 @@ impl AggregateVec { } } Self::SumU64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -138,9 +140,7 @@ impl AggregateVec { } } Self::SumF64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -150,9 +150,7 @@ impl AggregateVec { } } Self::MinU64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -162,9 +160,7 @@ impl AggregateVec { } } Self::MinI64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -174,9 +170,7 @@ impl AggregateVec { } } Self::MinF64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -186,9 +180,7 @@ impl AggregateVec { } } Self::MinString(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -203,9 +195,7 @@ impl AggregateVec { } } Self::MinBytes(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -220,9 +210,7 @@ impl AggregateVec { } } Self::MinBool(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -232,9 +220,7 @@ impl AggregateVec { } } Self::MaxU64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -244,9 +230,7 @@ impl AggregateVec { } } Self::MaxI64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -256,9 +240,7 @@ impl AggregateVec { } } Self::MaxF64(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -268,9 +250,7 @@ impl AggregateVec { } } Self::MaxString(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -285,9 +265,7 @@ impl AggregateVec { } } Self::MaxBytes(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -302,9 +280,7 @@ impl AggregateVec { } } Self::MaxBool(arr) => { - if values.is_null(row_id) { - return; - } else if offset >= arr.len() { + if offset >= arr.len() { arr.resize(offset + 1, None); } @@ -524,6 +500,8 @@ impl AggregateVec { } } + /// Writes a textual representation of the value specified by `offset` to + /// the provided formatter. pub fn write_value(&self, offset: usize, f: &mut Formatter<'_>) -> std::fmt::Result { match self { Self::Count(arr) => match arr[offset] { @@ -642,64 +620,6 @@ impl AggregateVec { Ok(()) } - // Consumes self and returns the inner `Vec>`. - pub fn as_i64(&self) -> &Vec> { - match self { - Self::SumI64(arr) => arr, - Self::MinI64(arr) => arr, - Self::MaxI64(arr) => arr, - _ => panic!("cannot convert {} to Vec>", self), - } - } - - // Consumes self and returns the inner `Vec>`. - pub fn as_u64(&self) -> &Vec> { - match self { - Self::Count(arr) => arr, - Self::SumU64(arr) => arr, - Self::MinU64(arr) => arr, - Self::MaxU64(arr) => arr, - _ => panic!("cannot convert {} to Vec>", self), - } - } - - // Consumes self and returns the inner `Vec>`. - pub fn as_f64(&self) -> &Vec> { - match self { - Self::SumF64(arr) => arr, - Self::MinF64(arr) => arr, - Self::MaxF64(arr) => arr, - _ => panic!("cannot convert {} to Vec>", self), - } - } - - // Consumes self and returns the inner `Vec>`. - pub fn as_str(&self) -> &Vec> { - match self { - Self::MinString(arr) => arr, - Self::MaxString(arr) => arr, - _ => panic!("cannot convert {} to Vec>", self), - } - } - - // Consumes self and returns the inner `Vec>`. - pub fn as_bytes(&self) -> &Vec>> { - match self { - Self::MinBytes(arr) => arr, - Self::MaxBytes(arr) => arr, - _ => panic!("cannot convert {} to Vec>", self), - } - } - - // Consumes self and returns the inner `Vec>`. - pub fn as_bool(&self) -> &Vec> { - match self { - Self::MinBool(arr) => arr, - Self::MaxBool(arr) => arr, - _ => panic!("cannot convert {} to Vec", self), - } - } - // Consumes self and returns the inner `Vec>`. pub fn take_as_i64(self) -> Vec> { match self { @@ -758,7 +678,7 @@ impl AggregateVec { } } - /// Appends the `AggregateVec` with the provided `Option` iterator. + /// Extends the `AggregateVec` with the provided `Option` iterator. pub fn extend_with_i64(&mut self, itr: impl Iterator>) { match self { Self::SumI64(arr) => { @@ -774,7 +694,7 @@ impl AggregateVec { } } - /// Appends the `AggregateVec` with the provided `Option` iterator. + /// Extends the `AggregateVec` with the provided `Option` iterator. pub fn extend_with_u64(&mut self, itr: impl Iterator>) { match self { Self::Count(arr) => { @@ -793,7 +713,7 @@ impl AggregateVec { } } - /// Appends the `AggregateVec` with the provided `Option` iterator. + /// Extends the `AggregateVec` with the provided `Option` iterator. pub fn extend_with_f64(&mut self, itr: impl Iterator>) { match self { Self::SumF64(arr) => { @@ -809,7 +729,7 @@ impl AggregateVec { } } - /// Appends the `AggregateVec` with the provided `Option<&str>` iterator. + /// Extends the `AggregateVec` with the provided `Option<&str>` iterator. pub fn extend_with_str(&mut self, itr: impl Iterator>) { match self { Self::MinString(arr) => { @@ -822,7 +742,7 @@ impl AggregateVec { } } - /// Appends the `AggregateVec` with the provided `Option>` iterator. + /// Extends the `AggregateVec` with the provided `Option>` iterator. pub fn extend_with_bytes(&mut self, itr: impl Iterator>>) { match self { Self::MinBytes(arr) => { @@ -835,7 +755,7 @@ impl AggregateVec { } } - /// Appends the `AggregateVec` with the provided `Option<&[u8]>` iterator. + /// Extends the `AggregateVec` with the provided `Option<&[u8]>` iterator. pub fn extend_with_bool(&mut self, itr: impl Iterator>) { match self { Self::MinBool(arr) => { @@ -985,62 +905,37 @@ impl std::fmt::Display for AggregateVec { } } -impl From<(&AggregateType, &LogicalDataType, usize)> for AggregateVec { - fn from(v: (&AggregateType, &LogicalDataType, usize)) -> Self { - let length = v.2; +impl From<(&AggregateType, &LogicalDataType)> for AggregateVec { + fn from(v: (&AggregateType, &LogicalDataType)) -> Self { match (v.0, v.1) { - (AggregateType::Count, _) => Self::Count(vec![None; length]), - (AggregateType::First, LogicalDataType::Integer) => { - Self::FirstI64((vec![None; length], vec![None; length])) - } - (AggregateType::First, LogicalDataType::Unsigned) => { - Self::FirstU64((vec![None; length], vec![None; length])) - } - (AggregateType::First, LogicalDataType::Float) => { - Self::FirstF64((vec![None; length], vec![None; length])) - } - (AggregateType::First, LogicalDataType::String) => { - Self::FirstString((vec![None; length], vec![None; length])) - } - (AggregateType::First, LogicalDataType::Binary) => { - Self::FirstBytes((vec![None; length], vec![None; length])) - } - (AggregateType::First, LogicalDataType::Boolean) => { - Self::FirstBool((vec![None; length], vec![None; length])) - } - (AggregateType::Last, LogicalDataType::Integer) => { - Self::LastI64((vec![None; length], vec![None; length])) - } - (AggregateType::Last, LogicalDataType::Unsigned) => { - Self::LastU64((vec![None; length], vec![None; length])) - } - (AggregateType::Last, LogicalDataType::Float) => { - Self::LastF64((vec![None; length], vec![None; length])) - } - (AggregateType::Last, LogicalDataType::String) => { - Self::LastString((vec![None; length], vec![None; length])) - } - (AggregateType::Last, LogicalDataType::Binary) => { - Self::LastBytes((vec![None; length], vec![None; length])) - } - (AggregateType::Last, LogicalDataType::Boolean) => { - Self::LastBool((vec![None; length], vec![None; length])) - } - (AggregateType::Min, LogicalDataType::Integer) => Self::MinI64(vec![None; length]), - (AggregateType::Min, LogicalDataType::Unsigned) => Self::MinU64(vec![None; length]), - (AggregateType::Min, LogicalDataType::Float) => Self::MinF64(vec![None; length]), - (AggregateType::Min, LogicalDataType::String) => Self::MinString(vec![None; length]), - (AggregateType::Min, LogicalDataType::Binary) => Self::MinBytes(vec![None; length]), - (AggregateType::Min, LogicalDataType::Boolean) => Self::MinBool(vec![None; length]), - (AggregateType::Max, LogicalDataType::Integer) => Self::MaxI64(vec![None; length]), - (AggregateType::Max, LogicalDataType::Unsigned) => Self::MaxU64(vec![None; length]), - (AggregateType::Max, LogicalDataType::Float) => Self::MaxF64(vec![None; length]), - (AggregateType::Max, LogicalDataType::String) => Self::MaxString(vec![None; length]), - (AggregateType::Max, LogicalDataType::Binary) => Self::MaxBytes(vec![None; length]), - (AggregateType::Max, LogicalDataType::Boolean) => Self::MaxBool(vec![None; length]), - (AggregateType::Sum, LogicalDataType::Integer) => Self::SumI64(vec![None; length]), - (AggregateType::Sum, LogicalDataType::Unsigned) => Self::SumU64(vec![None; length]), - (AggregateType::Sum, LogicalDataType::Float) => Self::SumF64(vec![None; length]), + (AggregateType::Count, _) => Self::Count(vec![]), + (AggregateType::First, LogicalDataType::Integer) => Self::FirstI64((vec![], vec![])), + (AggregateType::First, LogicalDataType::Unsigned) => Self::FirstU64((vec![], vec![])), + (AggregateType::First, LogicalDataType::Float) => Self::FirstF64((vec![], vec![])), + (AggregateType::First, LogicalDataType::String) => Self::FirstString((vec![], vec![])), + (AggregateType::First, LogicalDataType::Binary) => Self::FirstBytes((vec![], vec![])), + (AggregateType::First, LogicalDataType::Boolean) => Self::FirstBool((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Integer) => Self::LastI64((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Unsigned) => Self::LastU64((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Float) => Self::LastF64((vec![], vec![])), + (AggregateType::Last, LogicalDataType::String) => Self::LastString((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Binary) => Self::LastBytes((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Boolean) => Self::LastBool((vec![], vec![])), + (AggregateType::Min, LogicalDataType::Integer) => Self::MinI64(vec![]), + (AggregateType::Min, LogicalDataType::Unsigned) => Self::MinU64(vec![]), + (AggregateType::Min, LogicalDataType::Float) => Self::MinF64(vec![]), + (AggregateType::Min, LogicalDataType::String) => Self::MinString(vec![]), + (AggregateType::Min, LogicalDataType::Binary) => Self::MinBytes(vec![]), + (AggregateType::Min, LogicalDataType::Boolean) => Self::MinBool(vec![]), + (AggregateType::Max, LogicalDataType::Integer) => Self::MaxI64(vec![]), + (AggregateType::Max, LogicalDataType::Unsigned) => Self::MaxU64(vec![]), + (AggregateType::Max, LogicalDataType::Float) => Self::MaxF64(vec![]), + (AggregateType::Max, LogicalDataType::String) => Self::MaxString(vec![]), + (AggregateType::Max, LogicalDataType::Binary) => Self::MaxBytes(vec![]), + (AggregateType::Max, LogicalDataType::Boolean) => Self::MaxBool(vec![]), + (AggregateType::Sum, LogicalDataType::Integer) => Self::SumI64(vec![]), + (AggregateType::Sum, LogicalDataType::Unsigned) => Self::SumU64(vec![]), + (AggregateType::Sum, LogicalDataType::Float) => Self::SumF64(vec![]), (AggregateType::Sum, _) => unreachable!("unsupported SUM aggregates"), } } @@ -2018,6 +1913,181 @@ impl EncodedValues { mod test { use super::*; + #[test] + fn aggregate_vec_update() { + // i64 + let values = Values::I64N(vec![Some(1), Some(2), Some(3), None, Some(-1), Some(2)]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::SumI64(vec![]), + AggregateVec::MinI64(vec![]), + AggregateVec::MaxI64(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(5)]), + AggregateVec::SumI64(vec![Some(7)]), + AggregateVec::MinI64(vec![Some(-1)]), + AggregateVec::MaxI64(vec![Some(3)]), + ] + ); + + // u64 + let values = Values::U64N(vec![Some(1), Some(2), Some(3), None, Some(0), Some(2)]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::SumU64(vec![]), + AggregateVec::MinU64(vec![]), + AggregateVec::MaxU64(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(5)]), + AggregateVec::SumU64(vec![Some(8)]), + AggregateVec::MinU64(vec![Some(0)]), + AggregateVec::MaxU64(vec![Some(3)]), + ] + ); + + // f64 + let values = Values::F64N(vec![ + Some(1.0), + Some(2.0), + Some(3.0), + None, + Some(0.0), + Some(2.0), + ]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::SumF64(vec![]), + AggregateVec::MinF64(vec![]), + AggregateVec::MaxF64(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(5)]), + AggregateVec::SumF64(vec![Some(8.0)]), + AggregateVec::MinF64(vec![Some(0.0)]), + AggregateVec::MaxF64(vec![Some(3.0)]), + ] + ); + + // string + let values = Values::String(vec![ + Some("Pop Song 89"), + Some("Orange Crush"), + Some("Stand"), + None, + ]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::MinString(vec![]), + AggregateVec::MaxString(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(3)]), + AggregateVec::MinString(vec![Some("Orange Crush".to_owned())]), + AggregateVec::MaxString(vec![Some("Stand".to_owned())]), + ] + ); + + // bytes + let arr = vec![ + "Pop Song 89".to_string(), + "Orange Crush".to_string(), + "Stand".to_string(), + ]; + let values = Values::ByteArray(vec![ + Some(arr[0].as_bytes()), + Some(arr[1].as_bytes()), + Some(arr[2].as_bytes()), + None, + ]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::MinBytes(vec![]), + AggregateVec::MaxBytes(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(3)]), + AggregateVec::MinBytes(vec![Some(arr[1].bytes().collect())]), + AggregateVec::MaxBytes(vec![Some(arr[2].bytes().collect())]), + ] + ); + + // bool + let values = Values::Bool(vec![Some(true), None, Some(false)]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::MinBool(vec![]), + AggregateVec::MaxBool(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(2)]), + AggregateVec::MinBool(vec![Some(false)]), + AggregateVec::MaxBool(vec![Some(true)]), + ] + ); + } + #[test] fn size() { let v1 = OwnedValue::Null; From 326016966fdb959c3f1fc7147a1698971a72af94 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 09:57:46 +0000 Subject: [PATCH 083/104] refactor: update read_buffer/src/row_group.rs Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- read_buffer/src/row_group.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index 6c3231c2dc..adcacff363 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -536,8 +536,7 @@ impl RowGroup { // These vectors will hold the decoded values of each part of each // group key. They are the output columns of the input columns used for // the grouping operation. - let mut group_cols_out: Vec>>> = vec![]; - group_cols_out.resize(groupby_encoded_ids.len(), vec![]); + let mut group_cols_out = vec![vec![]; groupby_encoded_ids.len()]; // Each of these vectors will be used to store each aggregate row-value // for a specific aggregate result column. From b44dc7d9a17c35c5e37d75e67fa0f2cde6f14f94 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 09:58:28 +0000 Subject: [PATCH 084/104] refactor: update read_buffer/src/value.rs Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- read_buffer/src/value.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index 2092c0d5da..84a2a32fe6 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -114,10 +114,7 @@ impl AggregateVec { arr.resize(offset + 1, None); } - match &mut arr[offset] { - Some(v) => *v += 1, - None => arr[offset] = Some(1), - } + *arr[offset].get_or_insert(0) += 1; } Self::SumI64(arr) => { if offset >= arr.len() { From ce55dd7ce3a840532aec96cd2b6e03f94ef7486b Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 10:01:02 +0000 Subject: [PATCH 085/104] refactor: update read_buffer/src/row_group.rs Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- read_buffer/src/row_group.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index adcacff363..684fafa8dd 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -1707,7 +1707,7 @@ impl<'row_group> ReadAggregateResult<'row_group> { &self.schema } - //9/ The number of rows in the result. + /// The number of rows in the result. pub fn rows(&self) -> usize { if self.aggregate_cols.is_empty() { return 0; From 3e9b2fff5c96c20c000ac292cbd84df8bfa8faaf Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 10:01:16 +0000 Subject: [PATCH 086/104] refactor: Update read_buffer/src/row_group.rs Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- read_buffer/src/row_group.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index 684fafa8dd..6ae8ab3f26 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -2026,12 +2026,12 @@ impl<'row_group> ReadAggregateResult<'row_group> { } } -// The `groupKey` struct is a wrapper over a specific row of data in grouping +// The `GroupKey` struct is a wrapper over a specific row of data in grouping // columns. // // Rather than pivot the columns into a row-wise orientation to sort them, we // can effectively sort a projection across them (`row_offset`) storing -// `groupKey`s in a vector and sorting that. +// `GroupKey`s in a vector and sorting that. struct GroupKey<'a> { columns: &'a [Vec>], row_offset: usize, From dfd39cfce610eadf50d61c64cab273be8067645f Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 14:32:20 +0000 Subject: [PATCH 087/104] refactor: apply suggestions from code review Co-authored-by: Andrew Lamb --- read_buffer/src/value.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index 84a2a32fe6..083c5055fd 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -6,6 +6,13 @@ use arrow_deps::arrow; use crate::{AggregateType, LogicalDataType}; #[derive(Clone, PartialEq, Debug)] +/// A type that holds aggregates where each variant encodes the underlying data +/// type and aggregate type for a vector of data. An `AggregateVec` can be +/// updated on a value-by-value basis and new values can be appended. +/// +/// The type is structured this way to improve the performance of aggregations +/// in the read buffer by reducing the number of matches (branches) needed per +/// row. pub enum AggregateVec { Count(Vec>), @@ -726,7 +733,7 @@ impl AggregateVec { } } - /// Extends the `AggregateVec` with the provided `Option<&str>` iterator. + /// Extends the `AggregateVec` with the provided `Option` iterator. pub fn extend_with_str(&mut self, itr: impl Iterator>) { match self { Self::MinString(arr) => { @@ -752,7 +759,7 @@ impl AggregateVec { } } - /// Extends the `AggregateVec` with the provided `Option<&[u8]>` iterator. + /// Extends the `AggregateVec` with the provided `Option` iterator. pub fn extend_with_bool(&mut self, itr: impl Iterator>) { match self { Self::MinBool(arr) => { @@ -1524,7 +1531,7 @@ impl<'a> Value<'a> { if let Self::Scalar(Scalar::F64(v)) = self { return v; } - panic!("cannot unwrap Value to u64"); + panic!("cannot unwrap Value to f64"); } pub fn str(self) -> &'a str { @@ -1763,7 +1770,7 @@ impl<'a> Values<'a> { } } - // Returns a value as a string. Panics if not possible. + // Returns a value as a &str. Panics if not possible. fn value_str(&self, i: usize) -> &'a str { match &self { Values::String(c) => c[i].unwrap(), From 6a2e5d69f1137a5f7f8b184aa3b6307f41353265 Mon Sep 17 00:00:00 2001 From: Nga Tran Date: Mon, 22 Mar 2021 15:50:42 -0400 Subject: [PATCH 088/104] chore: e2e test for unsigned types --- server/src/query_tests/scenarios.rs | 16 ++++++++++++ server/src/query_tests/sql.rs | 34 ++++++++++++++++++++++++++ server/src/query_tests/table_schema.rs | 18 ++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/server/src/query_tests/scenarios.rs b/server/src/query_tests/scenarios.rs index ca4bd9a868..427621606b 100644 --- a/server/src/query_tests/scenarios.rs +++ b/server/src/query_tests/scenarios.rs @@ -84,6 +84,22 @@ impl DBSetup for TwoMeasurements { } } +pub struct TwoMeasurementsUnsignedType {} +#[async_trait] +impl DBSetup for TwoMeasurementsUnsignedType { + async fn make(&self) -> Vec { + let partition_key = "1970-01-01T00"; + let lp_lines = vec![ + "restaurant,town=andover count=40000u 100", + "restaurant,town=reading count=632u 120", + "school,town=reading count=17u 150", + "school,town=andover count=25u 160", + ]; + + make_one_chunk_scenarios(partition_key, &lp_lines.join("\n")).await + } +} + /// Single measurement that has several different chunks with /// different (but compatible) schema pub struct MultiChunkSchemaMerge {} diff --git a/server/src/query_tests/sql.rs b/server/src/query_tests/sql.rs index 727853dca4..4c247a2313 100644 --- a/server/src/query_tests/sql.rs +++ b/server/src/query_tests/sql.rs @@ -143,6 +143,40 @@ async fn sql_select_with_schema_merge() { run_sql_test_case!(MultiChunkSchemaMerge {}, "SELECT * from cpu", &expected); } +#[tokio::test] +async fn sql_select_from_restaurant() { + let expected = vec![ + "+---------+-------+", + "| town | count |", + "+---------+-------+", + "| andover | 40000 |", + "| reading | 632 |", + "+---------+-------+", + ]; + run_sql_test_case!( + TwoMeasurementsUnsignedType {}, + "SELECT town, count from restaurant", + &expected + ); +} + +#[tokio::test] +async fn sql_select_from_school() { + let expected = vec![ + "+---------+-------+", + "| town | count |", + "+---------+-------+", + "| reading | 17 |", + "| andover | 25 |", + "+---------+-------+", + ]; + run_sql_test_case!( + TwoMeasurementsUnsignedType {}, + "SELECT town, count from school", + &expected + ); +} + #[tokio::test] async fn sql_select_with_schema_merge_subset() { let expected = vec![ diff --git a/server/src/query_tests/table_schema.rs b/server/src/query_tests/table_schema.rs index c3b49df0ae..7e68ff4eb5 100644 --- a/server/src/query_tests/table_schema.rs +++ b/server/src/query_tests/table_schema.rs @@ -113,3 +113,21 @@ async fn list_schema_disk_selection() { run_table_schema_test_case!(TwoMeasurements {}, selection, "disk", expected_schema); } + +#[tokio::test] +async fn list_schema_location_all() { + // we expect columns to come out in lexographic order by name + let expected_schema = SchemaBuilder::new() + .field("count", DataType::UInt64) + .timestamp() + .tag("town") + .build() + .unwrap(); + + run_table_schema_test_case!( + TwoMeasurementsUnsignedType {}, + Selection::All, + "restaurant", + expected_schema + ); +} From dea4f1c45187151217e6b40430603fac84f7950a Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 10:35:46 +0000 Subject: [PATCH 089/104] refactor: remove type from row_group --- read_buffer/src/row_group.rs | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index 6ae8ab3f26..19b05011b8 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -15,7 +15,7 @@ use crate::column::{cmp::Operator, Column, RowIDs, RowIDsOption}; use crate::schema; use crate::schema::{AggregateType, LogicalDataType, ResultSchema}; use crate::value::{ - AggregateResult, AggregateVec, EncodedValues, OwnedValue, Scalar, Value, Values, ValuesIterator, + AggregateVec, EncodedValues, OwnedValue, Scalar, Value, Values, ValuesIterator, }; use arrow_deps::arrow::record_batch::RecordBatch; use arrow_deps::{ @@ -1372,31 +1372,6 @@ impl TryFrom<&DfExpr> for BinaryExpr { } } -#[derive(Debug, PartialEq, Clone)] -pub struct AggregateResults<'row_group>(Vec>); - -impl<'row_group> AggregateResults<'row_group> { - fn len(&self) -> usize { - self.0.len() - } - - fn merge(&mut self, other: &AggregateResults<'row_group>) { - assert_eq!(self.0.len(), other.len()); - for (i, agg) in self.0.iter_mut().enumerate() { - agg.merge(&other.0[i]); - } - } -} - -impl<'a> IntoIterator for AggregateResults<'a> { - type Item = AggregateResult<'a>; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - // A representation of a column name. pub type ColumnName<'a> = &'a str; From dd0ab422d29609fa7bfa51de85da5a1e212ca897 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 10:45:09 +0000 Subject: [PATCH 090/104] refactor: remove redundant type --- read_buffer/src/column.rs | 70 -------- read_buffer/src/table.rs | 105 +----------- read_buffer/src/value.rs | 324 -------------------------------------- 3 files changed, 3 insertions(+), 496 deletions(-) diff --git a/read_buffer/src/column.rs b/read_buffer/src/column.rs index 7260754dc1..e2fddb5df0 100644 --- a/read_buffer/src/column.rs +++ b/read_buffer/src/column.rs @@ -1291,8 +1291,6 @@ mod test { use super::*; use arrow_deps::arrow::array::{Int64Array, StringArray}; - use crate::value::AggregateResult; - #[test] fn row_ids_intersect() { let mut row_ids = RowIDs::new_bitmap(); @@ -2190,74 +2188,6 @@ mod test { assert_eq!(col.count(&[0, 2][..]), 0); } - #[test] - fn aggregate_result() { - let mut res = AggregateResult::Count(0); - res.update(Value::Null); - assert!(matches!(res, AggregateResult::Count(0))); - res.update(Value::String("hello")); - assert!(matches!(res, AggregateResult::Count(1))); - - let mut res = AggregateResult::Min(Value::Null); - res.update(Value::String("Dance Yrself Clean")); - assert!(matches!( - res, - AggregateResult::Min(Value::String("Dance Yrself Clean")) - )); - res.update(Value::String("All My Friends")); - assert!(matches!( - res, - AggregateResult::Min(Value::String("All My Friends")) - )); - res.update(Value::String("Dance Yrself Clean")); - assert!(matches!( - res, - AggregateResult::Min(Value::String("All My Friends")) - )); - res.update(Value::Null); - assert!(matches!( - res, - AggregateResult::Min(Value::String("All My Friends")) - )); - - let mut res = AggregateResult::Max(Value::Null); - res.update(Value::Scalar(Scalar::I64(20))); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(20))) - )); - res.update(Value::Scalar(Scalar::I64(39))); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(39))) - )); - res.update(Value::Scalar(Scalar::I64(20))); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(39))) - )); - res.update(Value::Null); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(39))) - )); - - let mut res = AggregateResult::Sum(Scalar::Null); - res.update(Value::Null); - assert!(matches!(res, AggregateResult::Sum(Scalar::Null))); - res.update(Value::Scalar(Scalar::Null)); - assert!(matches!(res, AggregateResult::Sum(Scalar::Null))); - - res.update(Value::Scalar(Scalar::I64(20))); - assert!(matches!(res, AggregateResult::Sum(Scalar::I64(20)))); - - res.update(Value::Scalar(Scalar::I64(-5))); - assert!(matches!(res, AggregateResult::Sum(Scalar::I64(15)))); - - res.update(Value::Scalar(Scalar::Null)); - assert!(matches!(res, AggregateResult::Sum(Scalar::I64(15)))); - } - #[test] fn has_non_null_value() { // Check each column type is wired up. Actual logic is tested in encoders. diff --git a/read_buffer/src/table.rs b/read_buffer/src/table.rs index 23f630cf81..ba358f1655 100644 --- a/read_buffer/src/table.rs +++ b/read_buffer/src/table.rs @@ -12,7 +12,7 @@ use snafu::{ensure, Snafu}; use crate::row_group::{self, ColumnName, Predicate, RowGroup}; use crate::schema::{AggregateType, ColumnType, LogicalDataType, ResultSchema}; -use crate::value::{AggregateResult, Scalar, Value}; +use crate::value::Value; #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("cannot drop last row group in table; drop table"))] @@ -273,76 +273,15 @@ impl Table { _group_columns: Vec>, _aggregates: Vec<(ColumnName<'a>, AggregateType)>, _window: i64, - ) -> BTreeMap, Vec<(ColumnName<'a>, AggregateResult<'_>)>> { + ) -> BTreeMap, Vec<(ColumnName<'a>, ReadAggregateResults)>> { // identify segments where time range and predicates match could match // using segment meta data, and then execute against those segments and // merge results. todo!() } - // Perform aggregates without any grouping. Filtering on optional predicates - // and time range is still supported. - fn read_aggregate_no_group<'a>( - &self, - time_range: (i64, i64), - predicates: &[(&str, &str)], - aggregates: Vec<(ColumnName<'a>, AggregateType)>, - ) -> Vec<(ColumnName<'a>, AggregateResult<'_>)> { - // The fast path where there are no predicates or a time range to apply. - // We just want the equivalent of column statistics. - if predicates.is_empty() { - let mut results = Vec::with_capacity(aggregates.len()); - for (col_name, agg_type) in &aggregates { - match agg_type { - AggregateType::Count => { - results.push(( - col_name, - AggregateResult::Count(self.count(col_name, time_range)), - )); - } - AggregateType::First => { - results.push(( - col_name, - AggregateResult::First(self.first(col_name, time_range.0)), - )); - } - AggregateType::Last => { - results.push(( - col_name, - AggregateResult::Last(self.last(col_name, time_range.1)), - )); - } - AggregateType::Min => { - results.push(( - col_name, - AggregateResult::Min(self.min(col_name, time_range)), - )); - } - AggregateType::Max => { - results.push(( - col_name, - AggregateResult::Max(self.max(col_name, time_range)), - )); - } - AggregateType::Sum => { - let res = match self.sum(col_name, time_range) { - Some(x) => x, - None => Scalar::Null, - }; - - results.push((col_name, AggregateResult::Sum(res))); - } - } - } - } - - // Otherwise we have predicates so for each segment we will execute a - // generalised aggregation method and build up the result set. - todo!(); - } - // - // ---- Fast-path aggregations on single columns. + // ---- Fast-path first/last selectors. // // Returns the first value for the specified column across the table @@ -387,44 +326,6 @@ impl Table { todo!(); } - /// The minimum non-null value in the column for the table. - fn min(&self, _column_name: &str, _time_range: (i64, i64)) -> Value<'_> { - // Loop over segments, skipping any that don't satisfy the time range. - // Any segments completely overlapped can have a candidate min taken - // directly from their zone map. Partially overlapped segments will be - // read using the appropriate execution API. - // - // Return the min of minimums. - todo!(); - } - - /// The maximum non-null value in the column for the table. - fn max(&self, _column_name: &str, _time_range: (i64, i64)) -> Value<'_> { - // Loop over segments, skipping any that don't satisfy the time range. - // Any segments completely overlapped can have a candidate max taken - // directly from their zone map. Partially overlapped segments will be - // read using the appropriate execution API. - // - // Return the max of maximums. - todo!(); - } - - /// The number of non-null values in the column for the table. - fn count(&self, _column_name: &str, _time_range: (i64, i64)) -> u64 { - // Loop over segments, skipping any that don't satisfy the time range. - // Execute appropriate aggregation call on each segment and aggregate - // the results. - todo!(); - } - - /// The total sum of non-null values in the column for the table. - fn sum(&self, _column_name: &str, _time_range: (i64, i64)) -> Option { - // Loop over segments, skipping any that don't satisfy the time range. - // Execute appropriate aggregation call on each segment and aggregate - // the results. - todo!(); - } - // // ---- Schema API queries // diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index 083c5055fd..d3b1365d3a 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -945,330 +945,6 @@ impl From<(&AggregateType, &LogicalDataType)> for AggregateVec { } } -/// These variants hold aggregates, which are the results of applying aggregates -/// to column data. -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum AggregateResult<'a> { - // Any type of column can have rows counted. NULL values do not contribute - // to the count. If all rows are NULL then count will be `0`. - Count(u64), - - // Only numerical columns with scalar values can be summed. NULL values do - // not contribute to the sum, but if all rows are NULL then the sum is - // itself NULL (represented by `None`). - Sum(Scalar), - - // The minimum value in the column data. - Min(Value<'a>), - - // The maximum value in the column data. - Max(Value<'a>), - - // The first value in the column data and the corresponding timestamp. - First(Option<(i64, Value<'a>)>), - - // The last value in the column data and the corresponding timestamp. - Last(Option<(i64, Value<'a>)>), -} - -#[allow(unused_assignments)] -impl<'a> AggregateResult<'a> { - pub fn update(&mut self, other: Value<'a>) { - if other.is_null() { - // a NULL value has no effect on aggregates - return; - } - - match self { - Self::Count(v) => { - if !other.is_null() { - *v += 1; - } - } - Self::Min(v) => match (&v, &other) { - (Value::Null, _) => { - // something is always smaller than NULL - *v = other; - } - (Value::String(_), Value::Null) => {} // do nothing - (Value::String(a), Value::String(b)) => { - if a.cmp(b) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::String(a), Value::ByteArray(b)) => { - if a.as_bytes().cmp(b) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::ByteArray(_), Value::Null) => {} // do nothing - (Value::ByteArray(a), Value::String(b)) => { - if a.cmp(&b.as_bytes()) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::ByteArray(a), Value::ByteArray(b)) => { - if a.cmp(b) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::Scalar(_), Value::Null) => {} // do nothing - (Value::Scalar(a), Value::Scalar(b)) => { - if a > b { - *v = other; - } - } - (_, _) => unreachable!("not a possible variant combination"), - }, - Self::Max(v) => match (&v, &other) { - (Value::Null, _) => { - // something is always larger than NULL - *v = other; - } - (Value::String(_), Value::Null) => {} // do nothing - (Value::String(a), Value::String(b)) => { - if a.cmp(b) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::String(a), Value::ByteArray(b)) => { - if a.as_bytes().cmp(b) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::ByteArray(_), Value::Null) => {} // do nothing - (Value::ByteArray(a), Value::String(b)) => { - if a.cmp(&b.as_bytes()) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::ByteArray(a), Value::ByteArray(b)) => { - if a.cmp(b) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::Scalar(_), Value::Null) => {} // do nothing - (Value::Scalar(a), Value::Scalar(b)) => { - if a < b { - *v = other; - } - } - (_, _) => unreachable!("not a possible variant combination"), - }, - Self::Sum(v) => match (&v, &other) { - (Scalar::Null, Value::Scalar(other_scalar)) => { - // NULL + something == something - *v = *other_scalar; - } - (_, Value::Scalar(b)) => *v += b, - (_, _) => unreachable!("not a possible variant combination"), - }, - _ => unimplemented!("First and Last aggregates not implemented yet"), - } - } - - /// Merge `other` into `self` - pub fn merge(&mut self, other: &AggregateResult<'a>) { - match (self, other) { - (AggregateResult::Count(this), AggregateResult::Count(that)) => *this += *that, - (AggregateResult::Sum(this), AggregateResult::Sum(that)) => *this += that, - (AggregateResult::Min(this), AggregateResult::Min(that)) => { - if *this > *that { - *this = *that; - } - } - (AggregateResult::Max(this), AggregateResult::Max(that)) => { - if *this < *that { - *this = *that; - } - } - (a, b) => unimplemented!("merging {:?} into {:?} not yet implemented", b, a), - } - } - - pub fn try_as_str(&self) -> Option<&str> { - match &self { - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::String(s) => Some(s), - v => panic!("cannot convert {:?} to &str", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::String(s) => Some(s), - v => panic!("cannot convert {:?} to &str", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to &str"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to &str"), - AggregateResult::Sum(v) => panic!("cannot convert {:?} to &str", v), - AggregateResult::Count(_) => panic!("cannot convert count to &str"), - } - } - - pub fn try_as_bytes(&self) -> Option<&[u8]> { - match &self { - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::ByteArray(s) => Some(s), - v => panic!("cannot convert {:?} to &[u8]", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::ByteArray(s) => Some(s), - v => panic!("cannot convert {:?} to &[u8]", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to &[u8]"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to &[u8]"), - AggregateResult::Sum(v) => panic!("cannot convert {:?} to &[u8]", v), - AggregateResult::Count(_) => panic!("cannot convert count to &[u8]"), - } - } - - pub fn try_as_bool(&self) -> Option { - match &self { - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Boolean(s) => Some(*s), - v => panic!("cannot convert {:?} to bool", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Boolean(s) => Some(*s), - v => panic!("cannot convert {:?} to bool", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to bool"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to bool"), - AggregateResult::Sum(v) => panic!("cannot convert {:?} to bool", v), - AggregateResult::Count(_) => panic!("cannot convert count to bool"), - } - } - - pub fn try_as_i64_scalar(&self) -> Option { - match &self { - AggregateResult::Sum(v) => match v { - Scalar::Null => None, - Scalar::I64(v) => Some(*v), - v => panic!("cannot convert {:?} to i64", v), - }, - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::I64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to i64", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::I64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to i64", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to scalar"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to scalar"), - AggregateResult::Count(_) => panic!("cannot represent count as i64"), - } - } - - pub fn try_as_u64_scalar(&self) -> Option { - match &self { - AggregateResult::Sum(v) => match v { - Scalar::Null => None, - Scalar::U64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - AggregateResult::Count(c) => Some(*c), - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::U64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to u64", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::U64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to u64", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to scalar"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to scalar"), - } - } - - pub fn try_as_f64_scalar(&self) -> Option { - match &self { - AggregateResult::Sum(v) => match v { - Scalar::Null => None, - Scalar::F64(v) => Some(*v), - v => panic!("cannot convert {:?} to f64", v), - }, - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::F64(v) => Some(*v), - v => panic!("cannot convert {:?} to f64", v), - }, - v => panic!("cannot convert {:?} to f64", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::F64(v) => Some(*v), - v => panic!("cannot convert {:?} to f64", v), - }, - v => panic!("cannot convert {:?} to f64", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to scalar"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to scalar"), - AggregateResult::Count(_) => panic!("cannot represent count as f64"), - } - } -} - -impl From<&AggregateType> for AggregateResult<'_> { - fn from(typ: &AggregateType) -> Self { - match typ { - AggregateType::Count => Self::Count(0), - AggregateType::First => Self::First(None), - AggregateType::Last => Self::Last(None), - AggregateType::Min => Self::Min(Value::Null), - AggregateType::Max => Self::Max(Value::Null), - AggregateType::Sum => Self::Sum(Scalar::Null), - } - } -} - -impl std::fmt::Display for AggregateResult<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - AggregateResult::Count(v) => write!(f, "{}", v), - AggregateResult::First(v) => match v { - Some((_, v)) => write!(f, "{}", v), - None => write!(f, "NULL"), - }, - AggregateResult::Last(v) => match v { - Some((_, v)) => write!(f, "{}", v), - None => write!(f, "NULL"), - }, - AggregateResult::Min(v) => write!(f, "{}", v), - AggregateResult::Max(v) => write!(f, "{}", v), - AggregateResult::Sum(v) => write!(f, "{}", v), - } - } -} - /// A scalar is a numerical value that can be aggregated. #[derive(Debug, PartialEq, PartialOrd, Copy, Clone)] pub enum Scalar { From 300ac2e411ef0ebfe1441d42cc88787116a9afa4 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 10:51:03 +0000 Subject: [PATCH 091/104] refactor: simplify aggregate_columns --- read_buffer/src/row_group.rs | 46 +++++++++++++++--------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index 19b05011b8..f2fbbe36c8 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -924,38 +924,30 @@ impl RowGroup { }, }; - // References to the columns to be used as input for producing the - // output aggregates. Also returns the required aggregate type. - let input_aggregate_columns = dst + dst.aggregate_cols = dst .schema .aggregate_columns .iter() - .map(|(col_type, agg_type, _)| (self.column_by_name(col_type.as_str()), *agg_type)) - .collect::>(); + .map(|(col_type, agg_type, data_type)| { + let col = self.column_by_name(col_type.as_str()); // input aggregate column + let mut agg_vec = AggregateVec::from((agg_type, data_type)); - let mut output_aggregate_columns = dst - .schema - .aggregate_columns - .iter() - .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) + // produce single aggregate for the input column subject to a + // predicate filter. + match agg_type { + AggregateType::Count => { + let value = Value::Scalar(Scalar::U64(col.count(&row_ids) as u64)); + agg_vec.push(value); + } + AggregateType::First => unimplemented!("First not yet implemented"), + AggregateType::Last => unimplemented!("Last not yet implemented"), + AggregateType::Min => agg_vec.push(col.min(&row_ids)), + AggregateType::Max => agg_vec.push(col.max(&row_ids)), + AggregateType::Sum => agg_vec.push(Value::Scalar(col.sum(&row_ids))), + } + agg_vec + }) .collect::>(); - - for (i, (col, agg_type)) in input_aggregate_columns.iter().enumerate() { - match agg_type { - AggregateType::Count => { - let value = Value::Scalar(Scalar::U64(col.count(&row_ids) as u64)); - output_aggregate_columns[i].push(value); - } - AggregateType::First => unimplemented!("First not yet implemented"), - AggregateType::Last => unimplemented!("Last not yet implemented"), - AggregateType::Min => output_aggregate_columns[i].push(col.min(&row_ids)), - AggregateType::Max => output_aggregate_columns[i].push(col.max(&row_ids)), - AggregateType::Sum => { - output_aggregate_columns[i].push(Value::Scalar(col.sum(&row_ids))) - } - } - } - dst.aggregate_cols = output_aggregate_columns; } /// Given the predicate (which may be empty), determine a set of rows From cadc92dac976074f2dbe67e111c3c25df7932e20 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 11:10:11 +0000 Subject: [PATCH 092/104] refactor: remove dead code --- read_buffer/src/row_group.rs | 8 ++------ read_buffer/src/table.rs | 32 ++------------------------------ read_buffer/src/value.rs | 11 +---------- 3 files changed, 5 insertions(+), 46 deletions(-) diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index f2fbbe36c8..2a4f112bb0 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -894,7 +894,7 @@ impl RowGroup { // // In this case the rows are already in "group key order" and the aggregates // can be calculated by reading the rows in order. - fn read_group_sorted_stream( + fn _read_group_sorted_stream( &self, _predicates: &Predicate, _group_column: ColumnName<'_>, @@ -1146,6 +1146,7 @@ fn pack_u32_in_u128(packed_value: u128, encoded_id: u32, pos: usize) -> u128 { // Given a packed encoded group key, unpacks them into `n` individual `u32` // group keys, and stores them in `dst`. It is the caller's responsibility to // ensure n <= 4. +#[cfg(test)] fn unpack_u128_group_key(group_key_packed: u128, n: usize, mut dst: Vec) -> Vec { dst.resize(n, 0); @@ -1512,11 +1513,6 @@ impl MetaData { self.columns_size += column_size; } - // Returns meta information about the column. - fn column_meta(&self, name: ColumnName<'_>) -> &ColumnMeta { - self.columns.get(name).unwrap() - } - // Extract schema information for a set of columns. fn schema_for_column_names( &self, diff --git a/read_buffer/src/table.rs b/read_buffer/src/table.rs index ba358f1655..3ebf687e80 100644 --- a/read_buffer/src/table.rs +++ b/read_buffer/src/table.rs @@ -94,6 +94,8 @@ impl Table { row_groups.data.push(Arc::new(rg)); } + /// TODO(edd): wire up + /// /// Remove the row group at `position` from table, returning an error if the /// caller has attempted to drop the last row group. /// @@ -401,36 +403,6 @@ impl Table { Ok(dst) } - /// Determines if this table could satisfy the provided predicate. - /// - /// `false` is proof that no row within this table would match the - /// predicate, whilst `true` indicates one or more rows *might* match the - /// predicate. - fn could_satisfy_predicate(&self, predicate: &Predicate) -> bool { - // Get a snapshot of the table data under a read lock. - let (meta, row_groups) = { - let table_data = self.table_data.read().unwrap(); - (Arc::clone(&table_data.meta), table_data.data.to_vec()) - }; - - // if the table doesn't have a column for one of the predicate's - // expressions then the table cannot satisfy the predicate. - if !predicate - .iter() - .all(|expr| meta.columns.contains_key(expr.column())) - { - return false; - } - - // If there is a single row group in the table that could satisfy the - // predicate then the table itself could satisfy the predicate so return - // true. If none of the row groups could match then return false. - let exprs = predicate.expressions(); - row_groups - .iter() - .any(|row_group| row_group.could_satisfy_conjunctive_binary_expressions(exprs)) - } - /// Determines if this table contains one or more rows that satisfy the /// predicate. pub fn satisfies_predicate(&self, predicate: &Predicate) -> bool { diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index d3b1365d3a..c7adf11952 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeSet, convert::TryFrom, fmt::Formatter}; +use std::{convert::TryFrom, fmt::Formatter}; use std::{mem::size_of, sync::Arc}; use arrow_deps::arrow; @@ -1513,15 +1513,6 @@ impl<'a> Iterator for ValuesIterator<'a> { } } -#[derive(PartialEq, Debug)] -pub enum ValueSet<'a> { - // UTF-8 valid unicode strings - String(BTreeSet>), - - // Arbitrary collections of bytes - ByteArray(BTreeSet>), -} - #[derive(Debug, PartialEq)] /// A representation of encoded values for a column. pub enum EncodedValues { From cf51a1a3f1c3f0a0f4fc338e474afc72c84c7072 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Fri, 19 Mar 2021 10:19:13 +0100 Subject: [PATCH 093/104] feat: Add API for ShardConfig --- data_types/src/database_rules.rs | 269 +++++++++++++++++- generated_types/build.rs | 1 + .../influxdata/iox/management/v1/shard.proto | 68 +++++ 3 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 generated_types/protos/influxdata/iox/management/v1/shard.proto diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index 5e5013c952..517c126dbd 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -758,7 +758,7 @@ pub struct ShardConfig { /// Maps a matcher with specific target group. If the line/row matches /// it should be sent to the group. -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Default)] pub struct MatcherToTargets { pub matcher: Matcher, pub target: NodeGroup, @@ -781,7 +781,7 @@ pub struct HashRing { /// A matcher is used to match routing rules or subscriptions on a row-by-row /// (or line) basis. -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, Default)] pub struct Matcher { /// if provided, match if the table name matches against the regex #[serde(with = "serde_regex")] @@ -800,6 +800,147 @@ impl PartialEq for Matcher { } impl Eq for Matcher {} +impl From for management::ShardConfig { + fn from(shard_config: ShardConfig) -> Self { + Self { + specific_targets: shard_config.specific_targets.map(|i| i.into()), + hash_ring: shard_config.hash_ring.map(|i| i.into()), + ignore_errors: shard_config.ignore_errors, + } + } +} + +impl TryFrom for ShardConfig { + type Error = FieldViolation; + + fn try_from(proto: management::ShardConfig) -> Result { + Ok(Self { + specific_targets: proto + .specific_targets + .map(|i| i.try_into()) + .map_or(Ok(None), |r| r.map(Some))?, + hash_ring: proto + .hash_ring + .map(|i| i.try_into()) + .map_or(Ok(None), |r| r.map(Some))?, + ignore_errors: proto.ignore_errors, + }) + } +} + +/// Returns none if v matches its default value. +fn none_if_default(v: T) -> Option { + if v == Default::default() { + None + } else { + Some(v) + } +} + +impl From for management::MatcherToTargets { + fn from(matcher_to_targets: MatcherToTargets) -> Self { + Self { + matcher: none_if_default(matcher_to_targets.matcher.into()), + target: none_if_default(from_node_group_for_management_node_group( + matcher_to_targets.target, + )), + } + } +} + +impl TryFrom for MatcherToTargets { + type Error = FieldViolation; + + fn try_from(proto: management::MatcherToTargets) -> Result { + Ok(Self { + matcher: proto.matcher.unwrap_or_default().try_into()?, + target: try_from_management_node_group_for_node_group( + proto.target.unwrap_or_default(), + )?, + }) + } +} + +impl From for management::HashRing { + fn from(hash_ring: HashRing) -> Self { + Self { + table_name: hash_ring.table_name, + columns: hash_ring.columns, + node_groups: hash_ring + .node_groups + .into_iter() + .map(from_node_group_for_management_node_group) + .collect(), + } + } +} + +impl TryFrom for HashRing { + type Error = FieldViolation; + + fn try_from(proto: management::HashRing) -> Result { + Ok(Self { + table_name: proto.table_name, + columns: proto.columns, + node_groups: proto + .node_groups + .into_iter() + .map(try_from_management_node_group_for_node_group) + .collect::, _>>()?, + }) + } +} + +// cannot (and/or don't know how to) add impl From inside prost generated code +fn from_node_group_for_management_node_group(node_group: NodeGroup) -> management::NodeGroup { + management::NodeGroup { + nodes: node_group + .into_iter() + .map(|id| management::node_group::Node { id }) + .collect(), + } +} + +fn try_from_management_node_group_for_node_group( + proto: management::NodeGroup, +) -> Result { + Ok(proto.nodes.into_iter().map(|i| i.id).collect()) +} + +impl From for management::Matcher { + fn from(matcher: Matcher) -> Self { + Self { + table_name_regex: matcher + .table_name_regex + .map_or_else(|| "".into(), |r| r.to_string()), + predicate: matcher.predicate.unwrap_or_else(|| "".into()), + } + } +} + +impl TryFrom for Matcher { + type Error = FieldViolation; + + fn try_from(proto: management::Matcher) -> Result { + let table_name_regex = match &proto.table_name_regex as &str { + "" => None, + re => Some(Regex::new(re).map_err(|e| FieldViolation { + field: "table_name_regex".to_string(), + description: e.to_string(), + })?), + }; + let predicate = match proto.predicate { + p if p.is_empty() => None, + p => Some(p), + }; + + Ok(Self { + table_name_regex, + predicate, + }) + } +} + /// `PartitionId` is the object storage identifier for a specific partition. It /// should be a path that can be used against an object store to locate all the /// files and subdirectories for a partition. It takes the form of `/ = protobuf.clone().try_into(); + assert!(matcher.is_err()); + assert_eq!(matcher.err().unwrap().field, "table_name_regex"); + } + + #[test] + fn test_hash_ring_default() { + let protobuf = management::HashRing { + ..Default::default() + }; + + let hash_ring: HashRing = protobuf.clone().try_into().unwrap(); + let back: management::HashRing = hash_ring.clone().into(); + + assert_eq!(hash_ring.table_name, false); + assert_eq!(protobuf.table_name, back.table_name); + assert!(hash_ring.columns.is_empty()); + assert_eq!(protobuf.columns, back.columns); + assert!(hash_ring.node_groups.is_empty()); + assert_eq!(protobuf.node_groups, back.node_groups); + } + + #[test] + fn test_hash_ring_nodes() { + let protobuf = management::HashRing { + node_groups: vec![ + management::NodeGroup { + nodes: vec![ + management::node_group::Node { id: 10 }, + management::node_group::Node { id: 11 }, + management::node_group::Node { id: 12 }, + ], + }, + management::NodeGroup { + nodes: vec![management::node_group::Node { id: 20 }], + }, + ], + ..Default::default() + }; + + let hash_ring: HashRing = protobuf.clone().try_into().unwrap(); + + assert_eq!(hash_ring.node_groups.len(), 2); + assert_eq!(hash_ring.node_groups[0].len(), 3); + assert_eq!(hash_ring.node_groups[1].len(), 1); + } + + #[test] + fn test_matcher_to_targets_default() { + let protobuf = management::MatcherToTargets { + ..Default::default() + }; + + let matcher_to_targets: MatcherToTargets = protobuf.clone().try_into().unwrap(); + let back: management::MatcherToTargets = matcher_to_targets.clone().into(); + + assert_eq!( + matcher_to_targets.matcher, + Matcher { + ..Default::default() + } + ); + assert_eq!(protobuf.matcher, back.matcher); + + assert_eq!(matcher_to_targets.target, Vec::::new()); + assert_eq!(protobuf.target, back.target); + } + + #[test] + fn test_shard_config_default() { + let protobuf = management::ShardConfig { + ..Default::default() + }; + + let shard_config: ShardConfig = protobuf.clone().try_into().unwrap(); + let back: management::ShardConfig = shard_config.clone().into(); + + assert!(shard_config.specific_targets.is_none()); + assert_eq!(protobuf.specific_targets, back.specific_targets); + + assert!(shard_config.hash_ring.is_none()); + assert_eq!(protobuf.hash_ring, back.hash_ring); + + assert_eq!(shard_config.ignore_errors, false); + assert_eq!(protobuf.ignore_errors, back.ignore_errors); + } } diff --git a/generated_types/build.rs b/generated_types/build.rs index 38e65d59e0..cb5782aa69 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -37,6 +37,7 @@ fn generate_grpc_types(root: &Path) -> Result<()> { management_path.join("chunk.proto"), management_path.join("partition.proto"), management_path.join("service.proto"), + management_path.join("shard.proto"), management_path.join("jobs.proto"), write_path.join("service.proto"), root.join("grpc/health/v1/service.proto"), diff --git a/generated_types/protos/influxdata/iox/management/v1/shard.proto b/generated_types/protos/influxdata/iox/management/v1/shard.proto new file mode 100644 index 0000000000..eb081287d3 --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/shard.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + +// NOTE: documentation is manually synced from data_types/src/database_rules.rs + +// `ShardConfig` defines rules for assigning a line/row to an individual +// host or a group of hosts. A shard +// is a logical concept, but the usage is meant to split data into +// mutually exclusive areas. The rough order of organization is: +// database -> shard -> partition -> chunk. For example, you could shard +// based on table name and assign to 1 of 10 shards. Within each +// shard you would have partitions, which would likely be based off time. +// This makes it possible to horizontally scale out writes. +message ShardConfig { + /// An optional matcher. If there is a match, the route will be evaluated to + /// the given targets, otherwise the hash ring will be evaluated. This is + /// useful for overriding the hashring function on some hot spot. For + /// example, if you use the table name as the input to the hash function + /// and your ring has 4 slots. If two tables that are very hot get + /// assigned to the same slot you can override that by putting in a + /// specific matcher to pull that table over to a different node. + MatcherToTargets specific_targets = 1; + + /// An optional default hasher which will route to one in a collection of + /// nodes. + HashRing hash_ring = 2; + + /// If set to true the router will ignore any errors sent by the remote + /// targets in this route. That is, the write request will succeed + /// regardless of this route's success. + bool ignore_errors = 3; +} + +// Maps a matcher with specific target group. If the line/row matches +// it should be sent to the group. +message MatcherToTargets { + Matcher matcher = 1; + NodeGroup target = 2; +} + +/// A matcher is used to match routing rules or subscriptions on a row-by-row +/// (or line) basis. +message Matcher { + // if provided, match if the table name matches against the regex + string table_name_regex = 1; + // paul: what should we use for predicate matching here against a single row/line? + string predicate = 2; +} + +// A collection of IOx nodes +message NodeGroup { + message Node { + uint32 id = 1; + } + repeated Node nodes = 1; +} + +// HashRing is a rule for creating a hash key for a row and mapping that to +// an individual node on a ring. +message HashRing { + // If true the table name will be included in the hash key + bool table_name = 1; + // include the values of these columns in the hash key + repeated string columns = 2; + // ring of node groups. Each group holds a shard + repeated NodeGroup node_groups = 3; +} + From 8962236e0424704b746cd0bd42ded686d8a516c0 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Mon, 22 Mar 2021 23:35:36 +0100 Subject: [PATCH 094/104] fix: Update data_types/src/database_rules.rs Co-authored-by: Carol (Nichols || Goulding) <193874+carols10cents@users.noreply.github.com> --- data_types/src/database_rules.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index 517c126dbd..00e12f28bf 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -912,8 +912,9 @@ impl From for management::Matcher { Self { table_name_regex: matcher .table_name_regex - .map_or_else(|| "".into(), |r| r.to_string()), - predicate: matcher.predicate.unwrap_or_else(|| "".into()), + .map(|r| r.to_string()) + .unwrap_or_default(), + predicate: matcher.predicate.unwrap_or_default(), } } } @@ -1441,7 +1442,7 @@ mod tests { ..Default::default() }; - let matcher: Result = protobuf.clone().try_into(); + let matcher: Result = protobuf.try_into(); assert!(matcher.is_err()); assert_eq!(matcher.err().unwrap().field, "table_name_regex"); } @@ -1481,7 +1482,7 @@ mod tests { ..Default::default() }; - let hash_ring: HashRing = protobuf.clone().try_into().unwrap(); + let hash_ring: HashRing = protobuf.try_into().unwrap(); assert_eq!(hash_ring.node_groups.len(), 2); assert_eq!(hash_ring.node_groups[0].len(), 3); From 1cbfea7096cb494f3c2b004127d388f908da24af Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Tue, 23 Mar 2021 11:36:46 +0000 Subject: [PATCH 095/104] refactor: don't implement serialize for Db (#1037) --- server/src/db.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/server/src/db.rs b/server/src/db.rs index a3de6a84e3..01216e4dc3 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -16,7 +16,6 @@ use mutable_buffer::MutableBufferDb; use parking_lot::Mutex; use query::{Database, PartitionChunk}; use read_buffer::Database as ReadBufferDb; -use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use crate::buffer::Buffer; @@ -81,31 +80,26 @@ pub type Result = std::result::Result; const STARTING_SEQUENCE: u64 = 1; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug)] /// This is the main IOx Database object. It is the root object of any /// specific InfluxDB IOx instance pub struct Db { - #[serde(flatten)] pub rules: DatabaseRules, - #[serde(skip)] /// The (optional) mutable buffer stores incoming writes. If a /// database does not have a mutable buffer it can not accept /// writes (it is a read replica) pub mutable_buffer: Option, - #[serde(skip)] /// The read buffer holds chunk data in an in-memory optimized /// format. pub read_buffer: Arc, - #[serde(skip)] /// The wal buffer holds replicated writes in an append in-memory /// buffer. This buffer is used for sending data to subscribers /// and to persist segments in object storage for recovery. pub wal_buffer: Option>, - #[serde(skip)] sequence: AtomicU64, } impl Db { From d4eef65f2a101ca32bc52565f2f0ed2b91a77b8f Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Tue, 23 Mar 2021 12:34:55 +0000 Subject: [PATCH 096/104] feat: share job registry with Db struct (#1038) --- server/src/config.rs | 36 ++++++++++++++------ server/src/db.rs | 26 +++++++++------ server/src/lib.rs | 58 ++++++++++++++++++++++++--------- server/src/query_tests/utils.rs | 4 ++- server/src/snapshot.rs | 6 +++- 5 files changed, 93 insertions(+), 37 deletions(-) diff --git a/server/src/config.rs b/server/src/config.rs index 75d8f8b1ed..7778add316 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -1,5 +1,8 @@ -/// This module contains code for managing the configuration of the server. -use crate::{db::Db, Error, Result}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{Arc, RwLock}, +}; + use data_types::{ database_rules::{DatabaseRules, WriterId}, DatabaseName, @@ -8,22 +11,28 @@ use mutable_buffer::MutableBufferDb; use object_store::path::ObjectStorePath; use read_buffer::Database as ReadBufferDb; -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::{Arc, RwLock}, -}; +/// This module contains code for managing the configuration of the server. +use crate::{db::Db, Error, JobRegistry, Result}; pub(crate) const DB_RULES_FILE_NAME: &str = "rules.json"; /// The Config tracks the configuration od databases and their rules along /// with host groups for replication. It is used as an in-memory structure /// that can be loaded incrementally from objet storage. -#[derive(Default, Debug)] +#[derive(Debug)] pub(crate) struct Config { + jobs: Arc, state: RwLock, } impl Config { + pub(crate) fn new(jobs: Arc) -> Self { + Self { + state: Default::default(), + jobs, + } + } + pub(crate) fn create_db( &self, name: DatabaseName<'static>, @@ -45,7 +54,13 @@ impl Config { let read_buffer = ReadBufferDb::new(); let wal_buffer = rules.wal_buffer_config.as_ref().map(Into::into); - let db = Arc::new(Db::new(rules, mutable_buffer, read_buffer, wal_buffer)); + let db = Arc::new(Db::new( + rules, + mutable_buffer, + read_buffer, + wal_buffer, + Arc::clone(&self.jobs), + )); state.reservations.insert(name.clone()); Ok(CreateDatabaseHandle { @@ -147,13 +162,14 @@ impl<'a> Drop for CreateDatabaseHandle<'a> { #[cfg(test)] mod test { - use super::*; use object_store::{memory::InMemory, ObjectStore, ObjectStoreApi}; + use super::*; + #[test] fn create_db() { let name = DatabaseName::new("foo").unwrap(); - let config = Config::default(); + let config = Config::new(Arc::new(JobRegistry::new())); let rules = DatabaseRules::new(); { diff --git a/server/src/db.rs b/server/src/db.rs index 01216e4dc3..2ee81ae8ce 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -10,20 +10,20 @@ use std::{ }; use async_trait::async_trait; +use parking_lot::Mutex; +use snafu::{OptionExt, ResultExt, Snafu}; +use tracing::info; + +pub(crate) use chunk::DBChunk; use data_types::{chunk::ChunkSummary, database_rules::DatabaseRules}; use internal_types::{data::ReplicatedWrite, selection::Selection}; use mutable_buffer::MutableBufferDb; -use parking_lot::Mutex; use query::{Database, PartitionChunk}; use read_buffer::Database as ReadBufferDb; -use snafu::{OptionExt, ResultExt, Snafu}; -use crate::buffer::Buffer; - -use tracing::info; +use crate::{buffer::Buffer, JobRegistry}; mod chunk; -pub(crate) use chunk::DBChunk; pub mod pred; mod streams; @@ -100,6 +100,8 @@ pub struct Db { /// and to persist segments in object storage for recovery. pub wal_buffer: Option>, + jobs: Arc, + sequence: AtomicU64, } impl Db { @@ -108,6 +110,7 @@ impl Db { mutable_buffer: Option, read_buffer: ReadBufferDb, wal_buffer: Option, + jobs: Arc, ) -> Self { let wal_buffer = wal_buffer.map(Mutex::new); let read_buffer = Arc::new(read_buffer); @@ -116,6 +119,7 @@ impl Db { mutable_buffer, read_buffer, wal_buffer, + jobs, sequence: AtomicU64::new(STARTING_SEQUENCE), } } @@ -376,10 +380,6 @@ impl Database for Db { #[cfg(test)] mod tests { - use crate::query_tests::utils::make_db; - - use super::*; - use arrow_deps::{ arrow::record_batch::RecordBatch, assert_table_eq, datafusion::physical_plan::collect, }; @@ -392,6 +392,10 @@ mod tests { }; use test_helpers::assert_contains; + use crate::query_tests::utils::make_db; + + use super::*; + #[tokio::test] async fn write_no_mutable_buffer() { // Validate that writes are rejected if there is no mutable buffer @@ -592,6 +596,7 @@ mod tests { buffer_size: 300, ..Default::default() }; + let rules = DatabaseRules { mutable_buffer_config: Some(mbconf.clone()), ..Default::default() @@ -602,6 +607,7 @@ mod tests { Some(MutableBufferDb::new("foo")), read_buffer::Database::new(), None, // wal buffer + Arc::new(JobRegistry::new()), ); let mut writer = TestLPWriter::default(); diff --git a/server/src/lib.rs b/server/src/lib.rs index 69e5dfa3b6..103559200d 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -84,9 +84,8 @@ use data_types::{ job::Job, {DatabaseName, DatabaseNameError}, }; -use internal_types::data::{lines_to_replicated_write, ReplicatedWrite}; - use influxdb_line_protocol::ParsedLine; +use internal_types::data::{lines_to_replicated_write, ReplicatedWrite}; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use query::{exec::Executor, DatabaseStore}; @@ -95,7 +94,9 @@ use crate::{ object_store_path_for_database_config, Config, GRPCConnectionString, DB_RULES_FILE_NAME, }, db::Db, - tracker::{TrackedFutureExt, Tracker, TrackerId, TrackerRegistryWithHistory}, + tracker::{ + TrackedFutureExt, Tracker, TrackerId, TrackerRegistration, TrackerRegistryWithHistory, + }, }; pub mod buffer; @@ -149,9 +150,34 @@ pub enum Error { pub type Result = std::result::Result; -const STORE_ERROR_PAUSE_SECONDS: u64 = 100; const JOB_HISTORY_SIZE: usize = 1000; +/// The global job registry +#[derive(Debug)] +pub struct JobRegistry { + inner: Mutex>, +} + +impl Default for JobRegistry { + fn default() -> Self { + Self { + inner: Mutex::new(TrackerRegistryWithHistory::new(JOB_HISTORY_SIZE)), + } + } +} + +impl JobRegistry { + fn new() -> Self { + Default::default() + } + + pub fn register(&self, job: Job) -> (Tracker, TrackerRegistration) { + self.inner.lock().register(job) + } +} + +const STORE_ERROR_PAUSE_SECONDS: u64 = 100; + /// `Server` is the container struct for how servers store data internally, as /// well as how they communicate with other servers. Each server will have one /// of these structs, which keeps track of all replication and query rules. @@ -162,18 +188,20 @@ pub struct Server { connection_manager: Arc, pub store: Arc, executor: Arc, - jobs: Mutex>, + jobs: Arc, } impl Server { pub fn new(connection_manager: M, store: Arc) -> Self { + let jobs = Arc::new(JobRegistry::new()); + Self { id: AtomicU32::new(SERVER_ID_NOT_SET), - config: Arc::new(Config::default()), + config: Arc::new(Config::new(Arc::clone(&jobs))), store, connection_manager: Arc::new(connection_manager), executor: Arc::new(Executor::new()), - jobs: Mutex::new(TrackerRegistryWithHistory::new(JOB_HISTORY_SIZE)), + jobs, } } @@ -354,7 +382,7 @@ impl Server { let writer_id = self.require_id()?; let store = Arc::clone(&self.store); - let (_, tracker) = self.jobs.lock().register(Job::PersistSegment { + let (_, tracker) = self.jobs.register(Job::PersistSegment { writer_id, segment_id: segment.id, }); @@ -390,7 +418,7 @@ impl Server { } pub fn spawn_dummy_job(&self, nanos: Vec) -> Tracker { - let (tracker, registration) = self.jobs.lock().register(Job::Dummy { + let (tracker, registration) = self.jobs.register(Job::Dummy { nanos: nanos.clone(), }); @@ -422,7 +450,7 @@ impl Server { .db(&name) .context(DatabaseNotFound { db_name: &db_name })?; - let (tracker, registration) = self.jobs.lock().register(Job::CloseChunk { + let (tracker, registration) = self.jobs.register(Job::CloseChunk { db_name: db_name.clone(), partition_key: partition_key.clone(), chunk_id, @@ -467,12 +495,12 @@ impl Server { /// Returns a list of all jobs tracked by this server pub fn tracked_jobs(&self) -> Vec> { - self.jobs.lock().tracked() + self.jobs.inner.lock().tracked() } /// Returns a specific job tracked by this server pub fn get_job(&self, id: TrackerId) -> Option> { - self.jobs.lock().get(id) + self.jobs.inner.lock().get(id) } /// Background worker function @@ -480,7 +508,7 @@ impl Server { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); while !shutdown.is_cancelled() { - self.jobs.lock().reclaim(); + self.jobs.inner.lock().reclaim(); tokio::select! { _ = interval.tick() => {}, @@ -620,6 +648,8 @@ mod tests { use futures::TryStreamExt; use parking_lot::Mutex; use snafu::Snafu; + use tokio::task::JoinHandle; + use tokio_util::sync::CancellationToken; use arrow_deps::{assert_table_eq, datafusion::physical_plan::collect}; use data_types::database_rules::{ @@ -632,8 +662,6 @@ mod tests { use crate::buffer::Segment; use super::*; - use tokio::task::JoinHandle; - use tokio_util::sync::CancellationToken; type TestError = Box; type Result = std::result::Result; diff --git a/server/src/query_tests/utils.rs b/server/src/query_tests/utils.rs index e8ff457b8c..f55cd01463 100644 --- a/server/src/query_tests/utils.rs +++ b/server/src/query_tests/utils.rs @@ -1,7 +1,8 @@ use data_types::database_rules::DatabaseRules; use mutable_buffer::MutableBufferDb; -use crate::db::Db; +use crate::{db::Db, JobRegistry}; +use std::sync::Arc; /// Used for testing: create a Database with a local store pub fn make_db() -> Db { @@ -11,5 +12,6 @@ pub fn make_db() -> Db { Some(MutableBufferDb::new(name)), read_buffer::Database::new(), None, // wal buffer + Arc::new(JobRegistry::new()), ) } diff --git a/server/src/snapshot.rs b/server/src/snapshot.rs index de64050e16..76cdc88b71 100644 --- a/server/src/snapshot.rs +++ b/server/src/snapshot.rs @@ -361,7 +361,10 @@ impl TryClone for MemWriter { #[cfg(test)] mod tests { - use crate::db::{DBChunk, Db}; + use crate::{ + db::{DBChunk, Db}, + JobRegistry, + }; use read_buffer::Database as ReadBufferDb; use super::*; @@ -480,6 +483,7 @@ mem,host=A,region=west used=45 1 Some(MutableBufferDb::new(name)), ReadBufferDb::new(), None, // wal buffer + Arc::new(JobRegistry::new()), ) } } From f8f81a90f71033b22adc8f7cb5ed09311a131e97 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Tue, 23 Mar 2021 13:51:13 +0000 Subject: [PATCH 097/104] feat: sequence server shutdown (#1036) * feat: sequence server shutdown * chore: pr comments --- src/influxdb_ioxd.rs | 46 ++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index 1200729c0b..660d095355 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -2,7 +2,7 @@ use crate::commands::{ logging::LoggingLevel, run::{Config, ObjectStore as ObjStoreOpt}, }; -use futures::{pin_mut, FutureExt}; +use futures::{future::FusedFuture, pin_mut, FutureExt}; use hyper::server::conn::AddrIncoming; use object_store::{ self, aws::AmazonS3, azure::MicrosoftAzure, gcp::GoogleCloudStorage, ObjectStore, @@ -140,8 +140,11 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { warn!("server ID not set. ID must be set via the INFLUXDB_IOX_ID config or API before writing or querying data."); } - // Construct a token to trigger shutdown - let token = tokio_util::sync::CancellationToken::new(); + // An internal shutdown token for internal workers + let internal_shutdown = tokio_util::sync::CancellationToken::new(); + + // Construct a token to trigger shutdown of API services + let frontend_shutdown = internal_shutdown.child_token(); // Construct and start up gRPC server let grpc_bind_addr = config.grpc_bind_address; @@ -149,21 +152,23 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { .await .context(StartListeningGrpc { grpc_bind_addr })?; - let grpc_server = rpc::serve(socket, Arc::clone(&app_server), token.clone()).fuse(); + let grpc_server = rpc::serve(socket, Arc::clone(&app_server), frontend_shutdown.clone()).fuse(); info!(bind_address=?grpc_bind_addr, "gRPC server listening"); let bind_addr = config.http_bind_address; let addr = AddrIncoming::bind(&bind_addr).context(StartListeningHttp { bind_addr })?; - let http_server = http::serve(addr, Arc::clone(&app_server), token.clone()).fuse(); + let http_server = http::serve(addr, Arc::clone(&app_server), frontend_shutdown.clone()).fuse(); info!(bind_address=?bind_addr, "HTTP server listening"); let git_hash = option_env!("GIT_HASH").unwrap_or("UNKNOWN"); info!(git_hash, "InfluxDB IOx server ready"); // Get IOx background worker task - let app = app_server.background_worker(token.clone()).fuse(); + let background_worker = app_server + .background_worker(internal_shutdown.clone()) + .fuse(); // Shutdown signal let signal = wait_for_signal().fuse(); @@ -192,19 +197,29 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { // pin_mut constructs a Pin<&mut T> from a T by preventing moving the T // from the current stack frame and constructing a Pin<&mut T> to it pin_mut!(signal); - pin_mut!(app); + pin_mut!(background_worker); pin_mut!(grpc_server); pin_mut!(http_server); // Return the first error encountered let mut res = Ok(()); - // Trigger graceful shutdown of server components on signal - // or any background service exiting - loop { + // Graceful shutdown can be triggered by sending SIGINT or SIGTERM to the + // process, or by a background task exiting - most likely with an error + // + // Graceful shutdown should then proceed in the following order + // 1. Stop accepting new HTTP and gRPC requests and drain existing connections + // 2. Trigger shutdown of internal background workers loops + // + // This is important to ensure background tasks, such as polling the tracker + // registry, don't exit before HTTP and gRPC requests dependent on them + while !grpc_server.is_terminated() && !http_server.is_terminated() { futures::select! { _ = signal => info!("Shutdown requested"), - _ = app => info!("Background worker shutdown"), + _ = background_worker => { + info!("background worker shutdown prematurely"); + internal_shutdown.cancel(); + }, result = grpc_server => match result { Ok(_) => info!("gRPC server shutdown"), Err(error) => { @@ -219,13 +234,16 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { res = res.and(Err(Error::ServingHttp{source: error})) } }, - complete => break } - token.cancel() + frontend_shutdown.cancel() } - info!("InfluxDB IOx server completed shutting down"); + info!("frontend shutdown completed"); + internal_shutdown.cancel(); + background_worker.await; + + info!("server completed shutting down"); res } From 585213e51f86fdde401402a902b2fecebd5424f7 Mon Sep 17 00:00:00 2001 From: Edd Robinson Date: Mon, 22 Mar 2021 21:19:47 +0000 Subject: [PATCH 098/104] refactor: return error for wrong group columns --- read_buffer/src/chunk.rs | 21 ++++++++-------- read_buffer/src/lib.rs | 10 ++++---- read_buffer/src/table.rs | 53 ++++++++++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/read_buffer/src/chunk.rs b/read_buffer/src/chunk.rs index 506b0d3338..e02d999e42 100644 --- a/read_buffer/src/chunk.rs +++ b/read_buffer/src/chunk.rs @@ -4,7 +4,7 @@ use std::{ }; use internal_types::selection::Selection; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use crate::row_group::RowGroup; use crate::row_group::{ColumnName, Predicate}; @@ -184,9 +184,7 @@ impl Chunk { let table = chunk_data .data .get(table_name) - .ok_or(Error::TableNotFound { - table_name: table_name.to_owned(), - })?; + .context(TableNotFound { table_name })?; Ok(table.read_filter(select_columns, predicate)) } @@ -195,7 +193,7 @@ impl Chunk { /// columns, optionally filtered by the provided predicate. Results are /// merged across all row groups within the returned table. /// - /// Returns `None` if the table no longer exists within the chunk. + /// Returns an error if the specified table does not exist. /// /// Note: `read_aggregate` currently only supports grouping on "tag" /// columns. @@ -205,17 +203,18 @@ impl Chunk { predicate: Predicate, group_columns: &Selection<'_>, aggregates: &[(ColumnName<'_>, AggregateType)], - ) -> Option { + ) -> Result { // read lock on chunk. let chunk_data = self.chunk_data.read().unwrap(); - // Lookup table by name and dispatch execution. - // - // TODO(edd): this should return an error - chunk_data + let table = chunk_data .data .get(table_name) - .map(|table| table.read_aggregate(predicate, group_columns, aggregates)) + .context(TableNotFound { table_name })?; + + table + .read_aggregate(predicate, group_columns, aggregates) + .context(TableError) } // diff --git a/read_buffer/src/lib.rs b/read_buffer/src/lib.rs index af21e2d92a..23d2372ee6 100644 --- a/read_buffer/src/lib.rs +++ b/read_buffer/src/lib.rs @@ -363,11 +363,11 @@ impl Database { // Get all relevant row groups for this chunk's table. This // is cheap because it doesn't execute the read operation, // but just gets references to the needed to data to do so. - if let Some(table_results) = - chunk.read_aggregate(table_name, predicate.clone(), &group_columns, &aggregates) - { - chunk_table_results.push(table_results); - } + let table_results = chunk + .read_aggregate(table_name, predicate.clone(), &group_columns, &aggregates) + .context(ChunkError)?; + + chunk_table_results.push(table_results); } Ok(ReadAggregateResults::new(chunk_table_results)) diff --git a/read_buffer/src/table.rs b/read_buffer/src/table.rs index 3ebf687e80..7e68a13857 100644 --- a/read_buffer/src/table.rs +++ b/read_buffer/src/table.rs @@ -228,7 +228,7 @@ impl Table { predicate: Predicate, group_columns: &'input Selection<'_>, aggregates: &'input [(ColumnName<'input>, AggregateType)], - ) -> ReadAggregateResults { + ) -> Result { let (meta, row_groups) = self.filter_row_groups(&predicate); // Filter out any column names that we do not have data for. @@ -241,13 +241,24 @@ impl Table { ..ResultSchema::default() }; + // Check all grouping columns are valid for grouping operation. + for (ct, _) in &schema.group_columns { + ensure!( + matches!(ct, ColumnType::Tag(_)), + UnsupportedColumnOperation { + msg: format!("column type must be ColumnType::Tag, got {:?}", ct), + column_name: ct.as_str().to_string(), + }, + ) + } + // return the iterator to build the results. - ReadAggregateResults { + Ok(ReadAggregateResults { schema, predicate, row_groups, ..Default::default() - } + }) } /// Returns aggregates segmented by grouping keys and windowed by time. @@ -1064,11 +1075,13 @@ mod test { table.add_row_group(rg); // no predicate aggregate - let mut results = table.read_aggregate( - Predicate::default(), - &Selection::Some(&[]), - &[("time", AggregateType::Count), ("time", AggregateType::Sum)], - ); + let mut results = table + .read_aggregate( + Predicate::default(), + &Selection::Some(&[]), + &[("time", AggregateType::Count), ("time", AggregateType::Sum)], + ) + .unwrap(); // check the column result schema let exp_schema = ResultSchema { @@ -1095,17 +1108,31 @@ mod test { assert!(matches!(results.next_merged_result(), None)); // apply a predicate - let mut results = table.read_aggregate( - Predicate::new(vec![BinaryExpr::from(("region", "=", "west"))]), - &Selection::Some(&[]), - &[("time", AggregateType::Count), ("time", AggregateType::Sum)], - ); + let mut results = table + .read_aggregate( + Predicate::new(vec![BinaryExpr::from(("region", "=", "west"))]), + &Selection::Some(&[]), + &[("time", AggregateType::Count), ("time", AggregateType::Sum)], + ) + .unwrap(); assert_eq!( DisplayReadAggregateResults(vec![results.next_merged_result().unwrap()]).to_string(), "time_count,time_sum\n2,300\n", ); assert!(matches!(results.next_merged_result(), None)); + + // group on wrong columns. + let results = table.read_aggregate( + Predicate::new(vec![BinaryExpr::from(("region", "=", "west"))]), + &Selection::Some(&["time"]), + &[("min", AggregateType::Min)], + ); + + assert!(matches!( + &results, + Err(Error::UnsupportedColumnOperation { .. }) + ),); } #[test] From 13d361e6452ae8747f57e30160852796a736ad27 Mon Sep 17 00:00:00 2001 From: Nga Tran Date: Tue, 23 Mar 2021 15:35:00 -0400 Subject: [PATCH 099/104] feat: support sicientific float data type --- influxdb_line_protocol/src/lib.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/influxdb_line_protocol/src/lib.rs b/influxdb_line_protocol/src/lib.rs index 9ede7fd3f0..695134e109 100644 --- a/influxdb_line_protocol/src/lib.rs +++ b/influxdb_line_protocol/src/lib.rs @@ -1470,6 +1470,21 @@ mod test { Ok(()) } + #[test] + fn parse_scientific_float() -> Result { + let input = "m0 field=-1.234456e+06 1615869152385000000"; + //let input = "m0 field=10"; + let parsed = parse(input); + + assert!( + matches!(parsed, Err(super::Error::CannotParseEntireLine { .. })), + "Wrong error: {:?}", + parsed, + ); + + Ok(()) + } + #[test] fn parse_negative_float() -> Result { let input = "m0 field2=-1 99"; From b758804df795cf0420b6154a286cf9a5b335d60a Mon Sep 17 00:00:00 2001 From: Nga Tran Date: Tue, 23 Mar 2021 18:09:12 -0400 Subject: [PATCH 100/104] feat: add functions to suport scientific notations --- influxdb_line_protocol/src/lib.rs | 43 ++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/influxdb_line_protocol/src/lib.rs b/influxdb_line_protocol/src/lib.rs index 695134e109..89ab451edb 100644 --- a/influxdb_line_protocol/src/lib.rs +++ b/influxdb_line_protocol/src/lib.rs @@ -675,7 +675,12 @@ fn field_uinteger_value(i: &str) -> IResult<&str, u64> { } fn field_float_value(i: &str) -> IResult<&str, f64> { - let value = alt((field_float_value_with_decimal, field_float_value_no_decimal)); + let value = alt(( + field_float_value_with_exponential_and_decimal, + field_float_value_with_exponential_no_decimal, + field_float_value_with_decimal, + field_float_value_no_decimal, + )); map_fail(value, |value| { value.parse().context(FloatValueInvalid { value }) })(i) @@ -685,6 +690,22 @@ fn field_float_value_with_decimal(i: &str) -> IResult<&str, &str> { recognize(separated_pair(integral_value_signed, tag("."), digit1))(i) } +fn field_float_value_with_exponential_and_decimal(i: &str) -> IResult<&str, &str> { + recognize(separated_pair( + integral_value_signed, + tag("."), + exponential_value, + ))(i) +} + +fn field_float_value_with_exponential_no_decimal(i: &str) -> IResult<&str, &str> { + exponential_value(i) +} + +fn exponential_value(i: &str) -> IResult<&str, &str> { + recognize(separated_pair(digit1, tag("e+"), digit1))(i) +} + fn field_float_value_no_decimal(i: &str) -> IResult<&str, &str> { integral_value_signed(i) } @@ -1473,16 +1494,30 @@ mod test { #[test] fn parse_scientific_float() -> Result { let input = "m0 field=-1.234456e+06 1615869152385000000"; - //let input = "m0 field=10"; - let parsed = parse(input); + let vals = parse(input)?; + assert_eq!(vals.len(), 1); + let input = "m0 field=1.234456e+06 1615869152385000000"; + let vals = parse(input)?; + assert_eq!(vals.len(), 1); + + let input = "m0 field=-1.234456e06 1615869152385000000"; + let parsed = parse(input); assert!( matches!(parsed, Err(super::Error::CannotParseEntireLine { .. })), "Wrong error: {:?}", parsed, ); - Ok(()) + let input = "m0 field=1.234456e06 1615869152385000000"; + let parsed = parse(input); + assert!( + matches!(parsed, Err(super::Error::CannotParseEntireLine { .. })), + "Wrong error: {:?}", + parsed, + ); + + Ok(()) } #[test] From 360df39d4c82c6e6625e8580e1226e5b2fe9db81 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Wed, 24 Mar 2021 09:43:52 +0000 Subject: [PATCH 101/104] feat: per-Db background tasks (#1032) * feat: per-Db background tasks (#1007) * chore: review comments * fix: ensure tracker registry is drained on shutdown * chore: formatting * chore: further review fixes Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- server/src/config.rs | 140 ++++++++++++++++++++++++++++++--- server/src/db.rs | 29 ++++++- server/src/lib.rs | 37 ++++++++- server/src/tracker/history.rs | 4 + server/src/tracker/registry.rs | 5 ++ src/influxdb_ioxd.rs | 13 +-- 6 files changed, 210 insertions(+), 18 deletions(-) diff --git a/server/src/config.rs b/server/src/config.rs index 7778add316..4082203faa 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -13,14 +13,23 @@ use read_buffer::Database as ReadBufferDb; /// This module contains code for managing the configuration of the server. use crate::{db::Db, Error, JobRegistry, Result}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tracing::{error, info, warn, Instrument}; pub(crate) const DB_RULES_FILE_NAME: &str = "rules.json"; -/// The Config tracks the configuration od databases and their rules along +/// The Config tracks the configuration of databases and their rules along /// with host groups for replication. It is used as an in-memory structure -/// that can be loaded incrementally from objet storage. +/// that can be loaded incrementally from object storage. +/// +/// drain() should be called prior to drop to ensure termination +/// of background worker tasks. They will be cancelled on drop +/// but they are effectively "detached" at that point, and they may not +/// run to completion if the tokio runtime is dropped #[derive(Debug)] pub(crate) struct Config { + shutdown: CancellationToken, jobs: Arc, state: RwLock, } @@ -28,6 +37,7 @@ pub(crate) struct Config { impl Config { pub(crate) fn new(jobs: Arc) -> Self { Self { + shutdown: Default::default(), state: Default::default(), jobs, } @@ -72,7 +82,7 @@ impl Config { pub(crate) fn db(&self, name: &DatabaseName<'_>) -> Option> { let state = self.state.read().expect("mutex poisoned"); - state.databases.get(name).cloned() + state.databases.get(name).map(|x| Arc::clone(&x.db)) } pub(crate) fn db_names_sorted(&self) -> Vec> { @@ -101,13 +111,64 @@ impl Config { .reservations .take(name) .expect("reservation doesn't exist"); - assert!(state.databases.insert(name, db).is_none()) + + if self.shutdown.is_cancelled() { + error!("server is shutting down"); + return; + } + + let shutdown = self.shutdown.child_token(); + let shutdown_captured = shutdown.clone(); + let db_captured = Arc::clone(&db); + let name_captured = name.clone(); + + let handle = Some(tokio::spawn(async move { + db_captured + .background_worker(shutdown_captured) + .instrument(tracing::info_span!("db_worker", database=%name_captured)) + .await + })); + + assert!(state + .databases + .insert( + name, + DatabaseState { + db, + handle, + shutdown + } + ) + .is_none()) } fn rollback(&self, name: &DatabaseName<'static>) { let mut state = self.state.write().expect("mutex poisoned"); state.reservations.remove(name); } + + /// Cancels and drains all background worker tasks + pub(crate) async fn drain(&self) { + info!("shutting down database background workers"); + + // This will cancel all background child tasks + self.shutdown.cancel(); + + let handles: Vec<_> = self + .state + .write() + .expect("mutex poisoned") + .databases + .iter_mut() + .filter_map(|(_, v)| v.join()) + .collect(); + + for handle in handles { + let _ = handle.await; + } + + info!("database background workers shutdown"); + } } pub fn object_store_path_for_database_config( @@ -126,12 +187,36 @@ pub type GRPCConnectionString = String; #[derive(Default, Debug)] struct ConfigState { reservations: BTreeSet>, - databases: BTreeMap, Arc>, + databases: BTreeMap, DatabaseState>, /// Map between remote IOx server IDs and management API connection strings. remotes: BTreeMap, } -/// CreateDatabaseHandle is retunred when a call is made to `create_db` on +#[derive(Debug)] +struct DatabaseState { + db: Arc, + handle: Option>, + shutdown: CancellationToken, +} + +impl DatabaseState { + fn join(&mut self) -> Option> { + self.handle.take() + } +} + +impl Drop for DatabaseState { + fn drop(&mut self) { + if self.handle.is_some() { + // Join should be called on `DatabaseState` prior to dropping, for example, by + // calling drain() on the owning `Config` + warn!("DatabaseState dropped without waiting for background task to complete"); + self.shutdown.cancel(); + } + } +} + +/// CreateDatabaseHandle is returned when a call is made to `create_db` on /// the Config struct. The handle can be used to hold a reservation for the /// database name. Calling `commit` on the handle will consume the struct /// and move the database from reserved to being in the config. @@ -166,8 +251,8 @@ mod test { use super::*; - #[test] - fn create_db() { + #[tokio::test] + async fn create_db() { let name = DatabaseName::new("foo").unwrap(); let config = Config::new(Arc::new(JobRegistry::new())); let rules = DatabaseRules::new(); @@ -181,8 +266,45 @@ mod test { let db_reservation = config.create_db(name.clone(), rules).unwrap(); db_reservation.commit(); assert!(config.db(&name).is_some()); + assert_eq!(config.db_names_sorted(), vec![name.clone()]); - assert_eq!(config.db_names_sorted(), vec![name]); + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + + assert!( + config + .db(&name) + .expect("expected database") + .worker_iterations() + > 0 + ); + + config.drain().await + } + + #[tokio::test] + async fn test_db_drop() { + let name = DatabaseName::new("foo").unwrap(); + let config = Config::new(Arc::new(JobRegistry::new())); + let rules = DatabaseRules::new(); + + let db_reservation = config.create_db(name.clone(), rules).unwrap(); + db_reservation.commit(); + + let token = config + .state + .read() + .expect("lock poisoned") + .databases + .get(&name) + .unwrap() + .shutdown + .clone(); + + // Drop config without calling drain + std::mem::drop(config); + + // This should cancel the the background task + assert!(token.is_cancelled()); } #[test] diff --git a/server/src/db.rs b/server/src/db.rs index 2ee81ae8ce..fd6a6c723d 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -4,7 +4,7 @@ use std::{ collections::BTreeMap, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, }, }; @@ -103,7 +103,10 @@ pub struct Db { jobs: Arc, sequence: AtomicU64, + + worker_iterations: AtomicUsize, } + impl Db { pub fn new( rules: DatabaseRules, @@ -121,6 +124,7 @@ impl Db { wal_buffer, jobs, sequence: AtomicU64::new(STARTING_SEQUENCE), + worker_iterations: AtomicUsize::new(0), } } @@ -314,6 +318,29 @@ impl Db { .chain(self.read_buffer_chunks(&partition_key).into_iter()) .map(|c| c.summary()) } + + /// Returns the number of iterations of the background worker loop + pub fn worker_iterations(&self) -> usize { + self.worker_iterations.load(Ordering::Relaxed) + } + + /// Background worker function + pub async fn background_worker(&self, shutdown: tokio_util::sync::CancellationToken) { + info!("started background worker"); + + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); + + while !shutdown.is_cancelled() { + self.worker_iterations.fetch_add(1, Ordering::Relaxed); + + tokio::select! { + _ = interval.tick() => {}, + _ = shutdown.cancelled() => break + } + } + + info!("finished background worker"); + } } impl PartialEq for Db { diff --git a/server/src/lib.rs b/server/src/lib.rs index 103559200d..18d0970024 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -77,7 +77,7 @@ use bytes::Bytes; use futures::stream::TryStreamExt; use parking_lot::Mutex; use snafu::{OptionExt, ResultExt, Snafu}; -use tracing::{debug, error, info}; +use tracing::{debug, error, info, warn}; use data_types::{ database_rules::{DatabaseRules, WriterId}, @@ -89,6 +89,8 @@ use internal_types::data::{lines_to_replicated_write, ReplicatedWrite}; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use query::{exec::Executor, DatabaseStore}; +use futures::{pin_mut, FutureExt}; + use crate::{ config::{ object_store_path_for_database_config, Config, GRPCConnectionString, DB_RULES_FILE_NAME, @@ -503,8 +505,10 @@ impl Server { self.jobs.inner.lock().get(id) } - /// Background worker function + /// Background worker function for the server pub async fn background_worker(&self, shutdown: tokio_util::sync::CancellationToken) { + info!("started background worker"); + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); while !shutdown.is_cancelled() { @@ -515,6 +519,35 @@ impl Server { _ = shutdown.cancelled() => break } } + + info!("shutting down background worker"); + + let join = self.config.drain().fuse(); + pin_mut!(join); + + // Keep running reclaim whilst shutting down in case something + // is waiting on a tracker to complete + loop { + self.jobs.inner.lock().reclaim(); + + futures::select! { + _ = interval.tick().fuse() => {}, + _ = join => break + } + } + + info!("draining tracker registry"); + + // Wait for any outstanding jobs to finish - frontend shutdown should be + // sequenced before shutting down the background workers and so there + // shouldn't be any + while self.jobs.inner.lock().tracked_len() != 0 { + self.jobs.inner.lock().reclaim(); + + interval.tick().await; + } + + info!("drained tracker registry"); } } diff --git a/server/src/tracker/history.rs b/server/src/tracker/history.rs index 29ec3b1fc3..7ffc462d70 100644 --- a/server/src/tracker/history.rs +++ b/server/src/tracker/history.rs @@ -32,6 +32,10 @@ impl TrackerRegistryWithHistory { } } + pub fn tracked_len(&self) -> usize { + self.registry.tracked_len() + } + /// Returns a list of trackers, including those that are no longer running pub fn tracked(&self) -> Vec> { let mut tracked = self.registry.tracked(); diff --git a/server/src/tracker/registry.rs b/server/src/tracker/registry.rs index 43ed019b0f..1c7996e2ad 100644 --- a/server/src/tracker/registry.rs +++ b/server/src/tracker/registry.rs @@ -97,6 +97,11 @@ impl TrackerRegistry { self.trackers.get(&id).map(|x| x.tracker.clone()) } + /// Returns the number of tracked tasks + pub fn tracked_len(&self) -> usize { + self.trackers.len() + } + /// Returns a list of trackers, including those that are no longer running pub fn tracked(&self) -> Vec> { self.trackers diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index 660d095355..6e21fd589d 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -11,7 +11,7 @@ use panic_logging::SendPanicsToTracing; use server::{ConnectionManagerImpl as ConnectionManager, Server as AppServer}; use snafu::{ResultExt, Snafu}; use std::{convert::TryFrom, fs, net::SocketAddr, path::PathBuf, sync::Arc}; -use tracing::{error, info, warn}; +use tracing::{error, info, warn, Instrument}; mod http; mod rpc; @@ -166,8 +166,9 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { info!(git_hash, "InfluxDB IOx server ready"); // Get IOx background worker task - let background_worker = app_server + let server_worker = app_server .background_worker(internal_shutdown.clone()) + .instrument(tracing::info_span!("server_worker")) .fuse(); // Shutdown signal @@ -197,7 +198,7 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { // pin_mut constructs a Pin<&mut T> from a T by preventing moving the T // from the current stack frame and constructing a Pin<&mut T> to it pin_mut!(signal); - pin_mut!(background_worker); + pin_mut!(server_worker); pin_mut!(grpc_server); pin_mut!(http_server); @@ -216,8 +217,8 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { while !grpc_server.is_terminated() && !http_server.is_terminated() { futures::select! { _ = signal => info!("Shutdown requested"), - _ = background_worker => { - info!("background worker shutdown prematurely"); + _ = server_worker => { + info!("server worker shutdown prematurely"); internal_shutdown.cancel(); }, result = grpc_server => match result { @@ -241,7 +242,7 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { info!("frontend shutdown completed"); internal_shutdown.cancel(); - background_worker.await; + server_worker.await; info!("server completed shutting down"); From 4911a014895f56ff44e5dda13c432a36e2f66925 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 23 Mar 2021 11:14:38 -0400 Subject: [PATCH 102/104] feat: Initial catalog objects --- Cargo.lock | 7 + Cargo.toml | 1 + catalog/Cargo.toml | 11 ++ catalog/src/chunk.rs | 69 +++++++ catalog/src/lib.rs | 416 +++++++++++++++++++++++++++++++++++++++ catalog/src/partition.rs | 23 +++ 6 files changed, 527 insertions(+) create mode 100644 catalog/Cargo.toml create mode 100644 catalog/src/chunk.rs create mode 100644 catalog/src/lib.rs create mode 100644 catalog/src/partition.rs diff --git a/Cargo.lock b/Cargo.lock index 7ee174bb1d..521c5b1d71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -433,6 +433,13 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "catalog" +version = "0.1.0" +dependencies = [ + "snafu", +] + [[package]] name = "cc" version = "1.0.67" diff --git a/Cargo.toml b/Cargo.toml index 89dc2b5d29..fea17d944c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ readme = "README.md" [workspace] # In alphabetical order members = [ "arrow_deps", + "catalog", "data_types", "generated_types", "google_types", diff --git a/catalog/Cargo.toml b/catalog/Cargo.toml new file mode 100644 index 0000000000..3b5fd318b5 --- /dev/null +++ b/catalog/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "catalog" +version = "0.1.0" +authors = ["Andrew Lamb "] +edition = "2018" +description = "InfluxDB IOx Metadata catalog implementation" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +snafu = "0.6" diff --git a/catalog/src/chunk.rs b/catalog/src/chunk.rs new file mode 100644 index 0000000000..ace990fb16 --- /dev/null +++ b/catalog/src/chunk.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +/// The state +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum ChunkState { + /// Chunk can accept new writes + Open, + + /// Chunk can still accept new writes, but will likely be closed soon + Closing, + + /// Chunk is closed for new writes and has become read only + Closed, + + /// Chunk is closed for new writes, and is actively moving to the read + /// buffer + Moving, + + /// Chunk has been completely loaded in the read buffer + Moved, +} + +/// The catalog representation of a Chunk in IOx. Note that a chunk +/// may exist in several physical locations at any given time (e.g. in +/// mutable buffer and in read buffer) +#[derive(Debug, PartialEq)] +pub struct Chunk { + /// What partition does the chunk belong to? + partition_key: Arc, + + /// The ID of the chunk + id: u32, + + /// The state of this chunk + state: ChunkState, + /* TODO: Additional fields + * such as object_store_path, etc */ +} + +impl Chunk { + /// Create a new chunk in the Open state + pub(crate) fn new(partition_key: impl Into, id: u32) -> Self { + let partition_key = Arc::new(partition_key.into()); + + Self { + partition_key, + id, + state: ChunkState::Open, + } + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn key(&self) -> &str { + self.partition_key.as_ref() + } + + pub fn state(&self) -> ChunkState { + self.state + } + + pub fn set_state(&mut self, state: ChunkState) { + // TODO add state transition validation here? + + self.state = state; + } +} diff --git a/catalog/src/lib.rs b/catalog/src/lib.rs new file mode 100644 index 0000000000..7fb9928190 --- /dev/null +++ b/catalog/src/lib.rs @@ -0,0 +1,416 @@ +//! This module contains the implementation of the InfluxDB IOx Metadata catalog +#![deny(rust_2018_idioms)] +#![warn( + missing_debug_implementations, + clippy::explicit_iter_loop, + clippy::use_self, + clippy::clone_on_ref_ptr +)] +use std::collections::{btree_map::Entry, BTreeMap}; + +use snafu::{OptionExt, Snafu}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("unknown partition: {}", partition_key))] + UnknownPartition { partition_key: String }, + + #[snafu(display("unknown chunk: {}:{}", partition_key, chunk_id))] + UnknownChunk { + partition_key: String, + chunk_id: u32, + }, + + #[snafu(display("partition already exists: {}", partition_key))] + PartitionAlreadyExists { partition_key: String }, + + #[snafu(display("chunk already exists: {}:{}", partition_key, chunk_id))] + ChunkAlreadyExists { + partition_key: String, + chunk_id: u32, + }, +} +pub type Result = std::result::Result; + +pub mod chunk; +pub mod partition; + +use chunk::Chunk; +use partition::Partition; + +/// InfluxDB IOx Metadata Catalog +/// +/// The Catalog stores information such as which chunks exist, what +/// state they are in, and what objects on object store are used, etc. +/// +/// The catalog is also responsible for (eventually) persisting this +/// information as well as ensuring that references between different +/// objects remain valid (e.g. that the `partition_key` field of all +/// Chunk's refer to valid partitions). +/// +/// +/// Note that the Partition does not "own" the Chunks (in the sense +/// there is not a list of chunks on the Partition object). This is +/// so that the Catalog carefully controls when objects are created / +/// removed from the catalog. Since the catalog can passing out +/// references to `Partition`, we don't want callers to be able to add +/// new chunks to partitions in any way other than calling +/// `Catalog::create_chunk` +#[derive(Debug, Default)] +pub struct Catalog { + /// The set of chunks in this database. The key is the partition + /// key, the values are the chunks for that partition, in some + /// arbitrary order + chunks: BTreeMap>, + + /// key is partition_key, value is Partition + partitions: BTreeMap, +} + +impl Catalog { + pub fn new() -> Self { + Self { + ..Default::default() + } + } + + /// Return an immutable chunk reference given the specified partition and + /// chunk id + pub fn chunk(&self, partition_key: impl AsRef, chunk_id: u32) -> Result<&Chunk> { + let partition_key = partition_key.as_ref(); + + self.partition_chunks(partition_key)? + .find(|c| c.id() == chunk_id) + .context(UnknownChunk { + partition_key, + chunk_id, + }) + } + + /// Return an mutable chunk reference given the specified partition and + /// chunk id + pub fn chunk_mut( + &mut self, + partition_key: impl AsRef, + chunk_id: u32, + ) -> Result<&mut Chunk> { + let partition_key = partition_key.as_ref(); + let chunks = self + .chunks + .get_mut(partition_key) + .context(UnknownPartition { partition_key })?; + + chunks + .iter_mut() + .find(|c| c.id() == chunk_id) + .context(UnknownChunk { + partition_key, + chunk_id, + }) + } + + /// Creates a new `Chunk` with id `id` within a specified Partition. + /// + /// This function also validates 'referential integrity' - aka + /// that the partition referred to by this chunk exists in the + /// catalog. + pub fn create_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { + let partition_key = partition_key.as_ref(); + let chunks = self + .chunks + .get_mut(partition_key) + .context(UnknownPartition { partition_key })?; + + // Ensure this chunk doesn't already exist + if chunks.iter().any(|c| c.id() == chunk_id) { + return ChunkAlreadyExists { + partition_key, + chunk_id, + } + .fail(); + } + + chunks.push(Chunk::new(partition_key, chunk_id)); + Ok(()) + } + + /// Removes the specified chunk from the catalog + pub fn drop_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { + let partition_key = partition_key.as_ref(); + let chunks = self + .chunks + .get_mut(partition_key) + .context(UnknownPartition { partition_key })?; + + let idx = chunks + .iter() + .enumerate() + .filter(|(_, c)| c.id() == chunk_id) + .map(|(i, _)| i) + .next() + .context(UnknownChunk { + partition_key, + chunk_id, + })?; + + chunks.remove(idx); + Ok(()) + } + + /// List all chunks in this database + pub fn chunks(&self) -> impl Iterator { + self.chunks.values().map(|chunks| chunks.iter()).flatten() + } + + /// List all chunks in a particular partition + pub fn partition_chunks( + &self, + partition_key: impl AsRef, + ) -> Result> { + let partition_key = partition_key.as_ref(); + let chunks = self + .chunks + .get(partition_key) + .context(UnknownPartition { partition_key })?; + + Ok(chunks.iter()) + } + + // List all partitions in this dataase + pub fn partitions(&self) -> impl Iterator { + self.partitions.values() + } + + // Get a specific partition by name + pub fn partition(&self, partition_key: impl AsRef) -> Option<&Partition> { + let partition_key = partition_key.as_ref(); + self.partitions.get(partition_key) + } + + // Create a new partition in the catalog, returning an error if it already + // exists + pub fn create_partition(&mut self, partition_key: impl Into) -> Result<()> { + let partition_key = partition_key.into(); + + let entry = self.partitions.entry(partition_key); + match entry { + Entry::Vacant(entry) => { + let chunks = self.chunks.insert(entry.key().to_string(), Vec::new()); + assert!(chunks.is_none()); // otherwise the structures are out of sync + + let partition = Partition::new(entry.key()); + entry.insert(partition); + Ok(()) + } + Entry::Occupied(entry) => PartitionAlreadyExists { + partition_key: entry.key(), + } + .fail(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn partition_create() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + + let err = catalog.create_partition("p1").unwrap_err(); + assert_eq!(err.to_string(), "partition already exists: p1"); + } + + #[test] + fn partition_get() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + catalog.create_partition("p2").unwrap(); + + let p1 = catalog.partition("p1").unwrap(); + assert_eq!(p1.key(), "p1"); + + let p2 = catalog.partition("p2").unwrap(); + assert_eq!(p2.key(), "p2"); + + let p3 = catalog.partition("p3"); + assert!(p3.is_none()); + } + + #[test] + fn partition_list() { + let mut catalog = Catalog::new(); + + assert_eq!(catalog.partitions().count(), 0); + + catalog.create_partition("p1").unwrap(); + catalog.create_partition("p2").unwrap(); + catalog.create_partition("p3").unwrap(); + + let mut partition_keys: Vec = + catalog.partitions().map(|p| p.key().into()).collect(); + partition_keys.sort_unstable(); + + assert_eq!(partition_keys, vec!["p1", "p2", "p3"]); + } + + #[test] + fn chunk_create_no_partition() { + let mut catalog = Catalog::new(); + let err = catalog + .create_chunk("non existent partition", 0) + .unwrap_err(); + assert_eq!(err.to_string(), "unknown partition: non existent partition"); + } + + #[test] + fn chunk_create() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + + let c1_0 = catalog.chunk("p1", 0).unwrap(); + assert_eq!(c1_0.key(), "p1"); + assert_eq!(c1_0.id(), 0); + + let c1_0 = catalog.chunk_mut("p1", 0).unwrap(); + assert_eq!(c1_0.key(), "p1"); + assert_eq!(c1_0.id(), 0); + + let c1_1 = catalog.chunk("p1", 1).unwrap(); + assert_eq!(c1_1.key(), "p1"); + assert_eq!(c1_1.id(), 1); + + let err = catalog.chunk("p3", 0).unwrap_err(); + assert_eq!(err.to_string(), "unknown partition: p3"); + + let err = catalog.chunk("p1", 100).unwrap_err(); + assert_eq!(err.to_string(), "unknown chunk: p1:100"); + } + + #[test] + fn chunk_create_dupe() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + + let res = catalog.create_chunk("p1", 0).unwrap_err(); + assert_eq!(res.to_string(), "chunk already exists: p1:0"); + } + + #[test] + fn chunk_list() { + let mut catalog = Catalog::new(); + assert_eq!(catalog.chunks().count(), 0); + + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + + catalog.create_partition("p2").unwrap(); + catalog.create_chunk("p2", 100).unwrap(); + + assert_eq!( + chunk_strings(&catalog), + vec!["Chunk p1:0", "Chunk p1:1", "Chunk p2:100"] + ); + + assert_eq!( + partition_chunk_strings(&catalog, "p1"), + vec!["Chunk p1:0", "Chunk p1:1"] + ); + assert_eq!( + partition_chunk_strings(&catalog, "p2"), + vec!["Chunk p2:100"] + ); + } + + #[test] + fn chunk_list_err() { + let catalog = Catalog::new(); + + match catalog.partition_chunks("p3") { + Err(err) => assert_eq!(err.to_string(), "unknown partition: p3"), + Ok(_) => panic!("unexpected success"), + }; + } + + fn chunk_strings(catalog: &Catalog) -> Vec { + let mut chunks: Vec = catalog + .chunks() + .map(|c| format!("Chunk {}:{}", c.key(), c.id())) + .collect(); + chunks.sort_unstable(); + + chunks + } + + fn partition_chunk_strings(catalog: &Catalog, partition_key: &str) -> Vec { + let mut chunks: Vec = catalog + .partition_chunks(partition_key) + .unwrap() + .map(|c| format!("Chunk {}:{}", c.key(), c.id())) + .collect(); + chunks.sort_unstable(); + + chunks + } + + #[test] + fn chunk_drop() { + let mut catalog = Catalog::new(); + + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + + catalog.create_partition("p2").unwrap(); + catalog.create_chunk("p2", 0).unwrap(); + + assert_eq!(catalog.chunks().count(), 3); + + catalog.drop_chunk("p1", 1).unwrap(); + catalog.chunk("p1", 1).unwrap_err(); // chunk is gone + assert_eq!(catalog.chunks().count(), 2); + + catalog.drop_chunk("p2", 0).unwrap(); + catalog.chunk("p2", 0).unwrap_err(); // chunk is gone + assert_eq!(catalog.chunks().count(), 1); + } + + #[test] + fn chunk_drop_non_existent_partition() { + let mut catalog = Catalog::new(); + let err = catalog.drop_chunk("p3", 0).unwrap_err(); + assert_eq!(err.to_string(), "unknown partition: p3"); + } + + #[test] + fn chunk_drop_non_existent_chunk() { + let mut catalog = Catalog::new(); + catalog.create_partition("p3").unwrap(); + + let err = catalog.drop_chunk("p3", 0).unwrap_err(); + assert_eq!(err.to_string(), "unknown chunk: p3:0"); + } + + #[test] + fn chunk_recreate_dropped() { + let mut catalog = Catalog::new(); + + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + assert_eq!(catalog.chunks().count(), 2); + + catalog.drop_chunk("p1", 0).unwrap(); + assert_eq!(catalog.chunks().count(), 1); + + // should be ok to recreate + catalog.create_chunk("p1", 0).unwrap(); + assert_eq!(catalog.chunks().count(), 2); + } +} diff --git a/catalog/src/partition.rs b/catalog/src/partition.rs new file mode 100644 index 0000000000..f88870f33f --- /dev/null +++ b/catalog/src/partition.rs @@ -0,0 +1,23 @@ +//! The catalog representation of a Partition + +#[derive(Debug, Default)] +/// IOx Catalog Partition +/// +/// A partition contains multiple Chunks. +pub struct Partition { + /// The partition key + key: String, +} + +impl Partition { + /// Create a new partition catalog object. + pub(crate) fn new(key: impl Into) -> Self { + let key = key.into(); + + Self { key } + } + + pub fn key(&self) -> &str { + &self.key + } +} From cdb5b64beb062f11577e0b91395b20f1d127b381 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 24 Mar 2021 07:15:05 -0400 Subject: [PATCH 103/104] refactor: store Chunks on Partitions --- catalog/src/lib.rs | 109 +++++++++++---------------------------- catalog/src/partition.rs | 101 +++++++++++++++++++++++++++++++++--- 2 files changed, 122 insertions(+), 88 deletions(-) diff --git a/catalog/src/lib.rs b/catalog/src/lib.rs index 7fb9928190..48fab9e2b7 100644 --- a/catalog/src/lib.rs +++ b/catalog/src/lib.rs @@ -47,23 +47,9 @@ use partition::Partition; /// information as well as ensuring that references between different /// objects remain valid (e.g. that the `partition_key` field of all /// Chunk's refer to valid partitions). -/// -/// -/// Note that the Partition does not "own" the Chunks (in the sense -/// there is not a list of chunks on the Partition object). This is -/// so that the Catalog carefully controls when objects are created / -/// removed from the catalog. Since the catalog can passing out -/// references to `Partition`, we don't want callers to be able to add -/// new chunks to partitions in any way other than calling -/// `Catalog::create_chunk` -#[derive(Debug, Default)] +#[derive(Default, Debug)] pub struct Catalog { - /// The set of chunks in this database. The key is the partition - /// key, the values are the chunks for that partition, in some - /// arbitrary order - chunks: BTreeMap>, - - /// key is partition_key, value is Partition + /// key is partition_key partitions: BTreeMap, } @@ -78,13 +64,7 @@ impl Catalog { /// chunk id pub fn chunk(&self, partition_key: impl AsRef, chunk_id: u32) -> Result<&Chunk> { let partition_key = partition_key.as_ref(); - - self.partition_chunks(partition_key)? - .find(|c| c.id() == chunk_id) - .context(UnknownChunk { - partition_key, - chunk_id, - }) + self.valid_partition(partition_key)?.chunk(chunk_id) } /// Return an mutable chunk reference given the specified partition and @@ -95,18 +75,7 @@ impl Catalog { chunk_id: u32, ) -> Result<&mut Chunk> { let partition_key = partition_key.as_ref(); - let chunks = self - .chunks - .get_mut(partition_key) - .context(UnknownPartition { partition_key })?; - - chunks - .iter_mut() - .find(|c| c.id() == chunk_id) - .context(UnknownChunk { - partition_key, - chunk_id, - }) + self.valid_partition_mut(partition_key)?.chunk_mut(chunk_id) } /// Creates a new `Chunk` with id `id` within a specified Partition. @@ -116,50 +85,20 @@ impl Catalog { /// catalog. pub fn create_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { let partition_key = partition_key.as_ref(); - let chunks = self - .chunks - .get_mut(partition_key) - .context(UnknownPartition { partition_key })?; - - // Ensure this chunk doesn't already exist - if chunks.iter().any(|c| c.id() == chunk_id) { - return ChunkAlreadyExists { - partition_key, - chunk_id, - } - .fail(); - } - - chunks.push(Chunk::new(partition_key, chunk_id)); - Ok(()) + self.valid_partition_mut(partition_key)? + .create_chunk(chunk_id) } /// Removes the specified chunk from the catalog pub fn drop_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { let partition_key = partition_key.as_ref(); - let chunks = self - .chunks - .get_mut(partition_key) - .context(UnknownPartition { partition_key })?; - - let idx = chunks - .iter() - .enumerate() - .filter(|(_, c)| c.id() == chunk_id) - .map(|(i, _)| i) - .next() - .context(UnknownChunk { - partition_key, - chunk_id, - })?; - - chunks.remove(idx); - Ok(()) + self.valid_partition_mut(partition_key)? + .drop_chunk(chunk_id) } /// List all chunks in this database pub fn chunks(&self) -> impl Iterator { - self.chunks.values().map(|chunks| chunks.iter()).flatten() + self.partitions.values().flat_map(|p| p.chunks()) } /// List all chunks in a particular partition @@ -168,12 +107,8 @@ impl Catalog { partition_key: impl AsRef, ) -> Result> { let partition_key = partition_key.as_ref(); - let chunks = self - .chunks - .get(partition_key) - .context(UnknownPartition { partition_key })?; - - Ok(chunks.iter()) + let iter = self.valid_partition(partition_key)?.chunks(); + Ok(iter) } // List all partitions in this dataase @@ -181,7 +116,8 @@ impl Catalog { self.partitions.values() } - // Get a specific partition by name + // Get a specific partition by name, returning `None` if there is no such + // partition pub fn partition(&self, partition_key: impl AsRef) -> Option<&Partition> { let partition_key = partition_key.as_ref(); self.partitions.get(partition_key) @@ -195,9 +131,6 @@ impl Catalog { let entry = self.partitions.entry(partition_key); match entry { Entry::Vacant(entry) => { - let chunks = self.chunks.insert(entry.key().to_string(), Vec::new()); - assert!(chunks.is_none()); // otherwise the structures are out of sync - let partition = Partition::new(entry.key()); entry.insert(partition); Ok(()) @@ -208,6 +141,22 @@ impl Catalog { .fail(), } } + + /// Internal helper to return the specified partition or an error + /// if there is no such partition + fn valid_partition(&self, partition_key: &str) -> Result<&Partition> { + self.partitions + .get(partition_key) + .context(UnknownPartition { partition_key }) + } + + /// Internal helper to return the specified partition as a mutable + /// reference or an error if there is no such partition + fn valid_partition_mut(&mut self, partition_key: &str) -> Result<&mut Partition> { + self.partitions + .get_mut(partition_key) + .context(UnknownPartition { partition_key }) + } } #[cfg(test)] diff --git a/catalog/src/partition.rs b/catalog/src/partition.rs index f88870f33f..2b0a4c6690 100644 --- a/catalog/src/partition.rs +++ b/catalog/src/partition.rs @@ -1,23 +1,108 @@ //! The catalog representation of a Partition -#[derive(Debug, Default)] +use crate::chunk::Chunk; +use std::collections::{btree_map::Entry, BTreeMap}; + +use super::{ChunkAlreadyExists, Result, UnknownChunk}; +use snafu::OptionExt; + /// IOx Catalog Partition /// /// A partition contains multiple Chunks. +#[derive(Debug, Default)] pub struct Partition { /// The partition key key: String, + + /// The chunks that make up this partition, indexed by id + chunks: BTreeMap, } impl Partition { - /// Create a new partition catalog object. - pub(crate) fn new(key: impl Into) -> Self { - let key = key.into(); - - Self { key } - } - + /// Return the partition_key of this Partition pub fn key(&self) -> &str { &self.key } } + +impl Partition { + /// Create a new partition catalog object. + /// + /// This function is not pub because `Partition`s should be + /// created using the interfaces on [`Catalog`] and not + /// instantiated directly. + pub(crate) fn new(key: impl Into) -> Self { + let key = key.into(); + + Self { + key, + ..Default::default() + } + } + + /// Create a new Chunk + /// + /// This function is not pub because `Chunks`s should be created + /// using the interfaces on [`Catalog`] and not instantiated + /// directly. + pub(crate) fn create_chunk(&mut self, chunk_id: u32) -> Result<()> { + let entry = self.chunks.entry(chunk_id); + match entry { + Entry::Vacant(entry) => { + entry.insert(Chunk::new(&self.key, chunk_id)); + Ok(()) + } + Entry::Occupied(_) => ChunkAlreadyExists { + partition_key: self.key(), + chunk_id, + } + .fail(), + } + } + + /// Drop the specified + /// + /// This function is not pub because `Chunks`s should be dropped + /// using the interfaces on [`Catalog`] and not instantiated + /// directly. + pub(crate) fn drop_chunk(&mut self, chunk_id: u32) -> Result<()> { + match self.chunks.remove(&chunk_id) { + Some(_) => Ok(()), + None => UnknownChunk { + partition_key: self.key(), + chunk_id, + } + .fail(), + } + } + + /// Return an immutable chunk reference by chunk id + /// + /// This function is not pub because `Chunks`s should be + /// accessed using the interfaces on [`Catalog`] + pub(crate) fn chunk(&self, chunk_id: u32) -> Result<&Chunk> { + self.chunks.get(&chunk_id).context(UnknownChunk { + partition_key: self.key(), + chunk_id, + }) + } + + /// Return a mutable chunk reference by chunk id + /// + /// This function is not pub because `Chunks`s should be + /// accessed using the interfaces on [`Catalog`] + pub(crate) fn chunk_mut(&mut self, chunk_id: u32) -> Result<&mut Chunk> { + self.chunks.get_mut(&chunk_id).context(UnknownChunk { + partition_key: &self.key, + chunk_id, + }) + } + + /// Return a iterator over chunks + /// + /// This function is not pub because `Chunks`s should be + /// accessed using the interfaces on [`Catalog`] + pub(crate) fn chunks(&self) -> impl Iterator { + self.chunks.values() + } +} From 90341b262aaf91539f8bf310399e98e5a66d3c78 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 24 Mar 2021 07:17:00 -0400 Subject: [PATCH 104/104] fix: improve comments --- catalog/src/lib.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/catalog/src/lib.rs b/catalog/src/lib.rs index 48fab9e2b7..d77d52a6dc 100644 --- a/catalog/src/lib.rs +++ b/catalog/src/lib.rs @@ -79,29 +79,25 @@ impl Catalog { } /// Creates a new `Chunk` with id `id` within a specified Partition. - /// - /// This function also validates 'referential integrity' - aka - /// that the partition referred to by this chunk exists in the - /// catalog. pub fn create_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { let partition_key = partition_key.as_ref(); self.valid_partition_mut(partition_key)? .create_chunk(chunk_id) } - /// Removes the specified chunk from the catalog + /// Removes the specified `Chunk` from the catalog pub fn drop_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { let partition_key = partition_key.as_ref(); self.valid_partition_mut(partition_key)? .drop_chunk(chunk_id) } - /// List all chunks in this database + /// List all `Chunk`s in this database pub fn chunks(&self) -> impl Iterator { self.partitions.values().flat_map(|p| p.chunks()) } - /// List all chunks in a particular partition + /// List all `Chunk`s in a particular partition pub fn partition_chunks( &self, partition_key: impl AsRef, @@ -111,7 +107,7 @@ impl Catalog { Ok(iter) } - // List all partitions in this dataase + // List all partitions in this database pub fn partitions(&self) -> impl Iterator { self.partitions.values() }