From 867aba847ac6e72b52bc6ac39027f47bed2d02b0 Mon Sep 17 00:00:00 2001 From: Dom Date: Mon, 30 Nov 2020 15:18:25 +0000 Subject: [PATCH] perf(convert): use flate2 for gzip decompression Switches from `libflate` to `flate2` for the top-level commands (specifically TSM conversion). --- Cargo.lock | 1 + Cargo.toml | 2 +- src/commands/input.rs | 10 +--------- src/server/http_routes.rs | 12 ++---------- 4 files changed, 5 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18330ea555..44fbf65dc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1350,6 +1350,7 @@ dependencies = [ "dirs 3.0.1", "dotenv", "env_logger", + "flate2", "futures", "generated_types", "hex", diff --git a/Cargo.toml b/Cargo.toml index 7bf2ac5876..ab98e7f97c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ tracing-futures="0.2.4" http = "0.2.0" snafu = "0.6.9" -libflate = "1.0.0" +flate2 = "1.0" [dev-dependencies] assert_cmd = "1.0.0" diff --git a/src/commands/input.rs b/src/commands/input.rs index fe51ef26ab..7885ae4e07 100644 --- a/src/commands/input.rs +++ b/src/commands/input.rs @@ -1,7 +1,6 @@ use arrow_deps::parquet::file::serialized_reader::{FileSource, SliceableCursor}; use ingest::parquet::ChunkReader; /// Module to handle input files (and maybe urls?) -use libflate::gzip; use packers::Name; use snafu::{ResultExt, Snafu}; use std::{ @@ -34,12 +33,6 @@ pub enum Error { source: io::Error, }, - #[snafu(display("Error creating decompressor for {} ({})", input_name.display(), source))] - UnableToCreateDecompressor { - input_name: PathBuf, - source: io::Error, - }, - #[snafu(display("Unknown input type: {} has an unknown input extension before .gz", input_name.display()))] UnknownInputTypeGzip { input_name: PathBuf }, @@ -234,8 +227,7 @@ impl InputReader { Some("gz") => { let buffer = || { let file = File::open(input_name).context(UnableToOpenInput { input_name })?; - let mut decoder = gzip::Decoder::new(file) - .context(UnableToCreateDecompressor { input_name })?; + let mut decoder = flate2::read::GzDecoder::new(file); let mut buffer = Vec::new(); decoder .read_to_end(&mut buffer) diff --git a/src/server/http_routes.rs b/src/server/http_routes.rs index 4a122c2210..c56704f6b0 100644 --- a/src/server/http_routes.rs +++ b/src/server/http_routes.rs @@ -116,9 +116,6 @@ pub enum ApplicationError { #[snafu(display("No handler for {:?} {}", method, path))] RouteNotFound { method: Method, path: String }, - - #[snafu(display("Internal error creating gzip decoder: {:?}", source))] - CreatingGzipDecoder { source: std::io::Error }, } impl ApplicationError { @@ -140,7 +137,6 @@ impl ApplicationError { Self::ParsingLineProtocol { .. } => StatusCode::BAD_REQUEST, Self::ReadingBodyAsGzip { .. } => StatusCode::BAD_REQUEST, Self::RouteNotFound { .. } => StatusCode::NOT_FOUND, - Self::CreatingGzipDecoder { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -190,12 +186,8 @@ async fn parse_body(req: hyper::Request) -> Result( struct ReadInfo { org: String, bucket: String, - // TODL This is currently a "SQL" request -- should be updated to conform + // TODO This is currently a "SQL" request -- should be updated to conform // to the V2 API for reading (using timestamps, etc). sql_query: String, }