Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor: Extract a cloud module to deal with cloud paths

pull/24376/head
Carol (Nichols || Goulding) 2021-01-14 10:18:43 -05:00
parent f8956dfbe8
commit ac22d2bb44
7 changed files with 92 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
object_store/src/aws.rs
View File

@ -1,7 +1,7 @@
//! This module contains the IOx implementation for using S3 as the object
//! store.
use crate::{
path::{CloudConverter, ObjectStorePath, DELIMITER},
path::{cloud::CloudConverter, ObjectStorePath, DELIMITER},
Error, ListResult, NoDataFromS3, ObjectMeta, Result, UnableToDeleteDataFromS3,
UnableToGetDataFromS3, UnableToGetPieceOfDataFromS3, UnableToPutDataToS3,
};

2
object_store/src/azure.rs
View File

@ -1,7 +1,7 @@
//! This module contains the IOx implementation for using Azure Blob storage as
//! the object store.
use crate::{
path::{CloudConverter, ObjectStorePath},
path::{cloud::CloudConverter, ObjectStorePath},
DataDoesNotMatchLength, Result, UnableToDeleteDataFromAzure, UnableToGetDataFromAzure,
UnableToListDataFromAzure, UnableToPutDataToAzure,
};

2
object_store/src/gcp.rs
View File

@ -1,7 +1,7 @@
//! This module contains the IOx implementation for using Google Cloud Storage
//! as the object store.
use crate::{
path::{CloudConverter, ObjectStorePath},
path::{cloud::CloudConverter, ObjectStorePath},
DataDoesNotMatchLength, Result, UnableToDeleteDataFromGcs, UnableToDeleteDataFromGcs2,
UnableToGetDataFromGcs, UnableToGetDataFromGcs2, UnableToListDataFromGcs,
UnableToListDataFromGcs2, UnableToPutDataToGcs,

2
object_store/src/lib.rs
View File

@ -152,7 +152,7 @@ impl ObjectStore {
use ObjectStoreIntegration::*;
match &self.0 {
AmazonS3(_) | GoogleCloudStorage(_) | InMemory(_) | MicrosoftAzure(_) => {
path::CloudConverter::convert(path)
path::cloud::CloudConverter::convert(path)
}
File(_) => path::FileConverter::convert(path).display().to_string(),
}

84
object_store/src/path.rs
View File

@ -1,10 +1,12 @@
//! This module contains code for abstracting object locations that work
//! across different backing implementations and platforms.
use itertools::Itertools;
use percent_encoding::{percent_decode_str, percent_encode, AsciiSet, CONTROLS};
use std::{mem, path::PathBuf};
/// Paths that came from or are to be used in cloud-based object storage
pub mod cloud;
/// Maximally processed storage-independent paths.
pub mod parsed;
use parsed::DirsAndFileName;
@ -207,43 +209,6 @@ impl PartialEq for PathRepresentation {
}
}
// TODO: I made these structs rather than functions because I could see
// `convert` being part of a trait, possibly, but that seemed a bit overly
// complex for now.
/// Converts `ObjectStorePath`s to `String`s that are appropriate for use as
/// locations in cloud storage.
#[derive(Debug, Clone, Copy)]
pub struct CloudConverter {}
impl CloudConverter {
/// Creates a cloud storage location by joining this `ObjectStorePath`'s
/// parts with `DELIMITER`
pub fn convert(object_store_path: &ObjectStorePath) -> String {
match &object_store_path.inner {
PathRepresentation::RawCloud(path) => path.to_owned(),
PathRepresentation::RawPathBuf(_path) => {
todo!("convert");
}
PathRepresentation::Parts(dirs_and_file_name) => {
let mut path = dirs_and_file_name
.directories
.iter()
.map(|p| &p.0)
.join(DELIMITER);
if !path.is_empty() {
path.push_str(DELIMITER);
}
if let Some(file_name) = &dirs_and_file_name.file_name {
path.push_str(&file_name.0);
}
path
}
}
}
}
/// Converts `ObjectStorePath`s to `String`s that are appropriate for use as
/// locations in filesystem storage.
#[derive(Debug, Clone, Copy)]
@ -385,49 +350,6 @@ mod tests {
// - Within a process, the same backing store will always be used
//
#[test]
fn cloud_prefix_no_trailing_delimiter_or_file_name() {
// Use case: a file named `test_file.json` exists in object storage and it
// should be returned for a search on prefix `test`, so the prefix path
// should not get a trailing delimiter automatically added
let mut prefix = ObjectStorePath::default();
prefix.set_file_name("test");
let converted = CloudConverter::convert(&prefix);
assert_eq!(converted, "test");
}
#[test]
fn cloud_prefix_with_trailing_delimiter() {
// Use case: files exist in object storage named `foo/bar.json` and
// `foo_test.json`. A search for the prefix `foo/` should return
// `foo/bar.json` but not `foo_test.json'.
let mut prefix = ObjectStorePath::default();
prefix.push_dir("test");
let converted = CloudConverter::convert(&prefix);
assert_eq!(converted, "test/");
}
#[test]
fn push_encodes() {
let mut location = ObjectStorePath::default();
location.push_dir("foo/bar");
location.push_dir("baz%2Ftest");
let converted = CloudConverter::convert(&location);
assert_eq!(converted, "foo%2Fbar/baz%252Ftest/");
}
#[test]
fn push_all_encodes() {
let mut location = ObjectStorePath::default();
location.push_all_dirs(&["foo/bar", "baz%2Ftest"]);
let converted = CloudConverter::convert(&location);
assert_eq!(converted, "foo%2Fbar/baz%252Ftest/");
}
#[test]
fn prefix_matches() {
let mut haystack = ObjectStorePath::default();

84
object_store/src/path/cloud.rs Normal file
View File

@ -0,0 +1,84 @@
use super::{ObjectStorePath, PathRepresentation, DELIMITER};
use itertools::Itertools;
/// Converts `ObjectStorePath`s to `String`s that are appropriate for use as
/// locations in cloud storage.
#[derive(Debug, Clone, Copy)]
pub struct CloudConverter {}
impl CloudConverter {
/// Creates a cloud storage location by joining this `ObjectStorePath`'s
/// parts with `DELIMITER`
pub fn convert(object_store_path: &ObjectStorePath) -> String {
match &object_store_path.inner {
PathRepresentation::RawCloud(path) => path.to_owned(),
PathRepresentation::RawPathBuf(_path) => {
todo!("convert");
}
PathRepresentation::Parts(dirs_and_file_name) => {
let mut path = dirs_and_file_name
.directories
.iter()
.map(|p| &p.0)
.join(DELIMITER);
if !path.is_empty() {
path.push_str(DELIMITER);
}
if let Some(file_name) = &dirs_and_file_name.file_name {
path.push_str(&file_name.0);
}
path
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cloud_prefix_no_trailing_delimiter_or_file_name() {
// Use case: a file named `test_file.json` exists in object storage and it
// should be returned for a search on prefix `test`, so the prefix path
// should not get a trailing delimiter automatically added
let mut prefix = ObjectStorePath::default();
prefix.set_file_name("test");
let converted = CloudConverter::convert(&prefix);
assert_eq!(converted, "test");
}
#[test]
fn cloud_prefix_with_trailing_delimiter() {
// Use case: files exist in object storage named `foo/bar.json` and
// `foo_test.json`. A search for the prefix `foo/` should return
// `foo/bar.json` but not `foo_test.json'.
let mut prefix = ObjectStorePath::default();
prefix.push_dir("test");
let converted = CloudConverter::convert(&prefix);
assert_eq!(converted, "test/");
}
#[test]
fn push_encodes() {
let mut location = ObjectStorePath::default();
location.push_dir("foo/bar");
location.push_dir("baz%2Ftest");
let converted = CloudConverter::convert(&location);
assert_eq!(converted, "foo%2Fbar/baz%252Ftest/");
}
#[test]
fn push_all_encodes() {
let mut location = ObjectStorePath::default();
location.push_all_dirs(&["foo/bar", "baz%2Ftest"]);
let converted = CloudConverter::convert(&location);
assert_eq!(converted, "foo%2Fbar/baz%252Ftest/");
}
}

2
server/src/buffer.rs
View File

@ -509,7 +509,7 @@ mod tests {
use super::*;
use data_types::{data::lines_to_replicated_write, database_rules::DatabaseRules};
use influxdb_line_protocol::parse_lines;
use object_store::path::CloudConverter;
use object_store::path::cloud::CloudConverter;
#[test]
fn append_increments_current_size_and_uses_existing_segment() {