Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor: get build

pull/24376/head
Edd Robinson 2021-05-26 22:40:39 +01:00
parent 04513e737d
commit 7ab27c4468
4 changed files with 87 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
read_buffer/src/chunk.rs
View File

@ -675,7 +675,7 @@ impl ChunkMetrics {
fn update_column_storage_statistics(&mut self, statistics: &[Statistics]) {
for stat in statistics {
let labels = &[
KeyValue::new("encoding", stat.enc_type),
KeyValue::new("encoding", stat.enc_type.clone()),
KeyValue::new("log_data_type", stat.log_data_type),
];
@ -689,7 +689,7 @@ impl ChunkMetrics {
self.column_raw_bytes_total.inc(
stat.raw_bytes - stat.raw_bytes_no_null,
&[
KeyValue::new("encoding", stat.enc_type),
KeyValue::new("encoding", stat.enc_type.clone()),
KeyValue::new("log_data_type", stat.log_data_type),
KeyValue::new("null", "true"),
],
@ -699,7 +699,7 @@ impl ChunkMetrics {
self.column_raw_bytes_total.inc(
stat.raw_bytes_no_null,
&[
KeyValue::new("encoding", stat.enc_type),
KeyValue::new("encoding", stat.enc_type.clone()),
KeyValue::new("log_data_type", stat.log_data_type),
KeyValue::new("null", "false"),
],
@ -709,7 +709,7 @@ impl ChunkMetrics {
self.column_values_total.inc(
stat.nulls as usize,
&[
KeyValue::new("encoding", stat.enc_type),
KeyValue::new("encoding", stat.enc_type.clone()),
KeyValue::new("log_data_type", stat.log_data_type),
KeyValue::new("null", "true"),
],
@ -719,7 +719,7 @@ impl ChunkMetrics {
self.column_values_total.inc(
(stat.values - stat.nulls) as usize,
&[
KeyValue::new("encoding", stat.enc_type),
KeyValue::new("encoding", stat.enc_type.clone()),
KeyValue::new("log_data_type", stat.log_data_type),
KeyValue::new("null", "false"),
],
@ -977,78 +977,90 @@ mod test {
assert_eq!(
String::from_utf8(reg.registry().metrics_as_text()).unwrap(),
vec![
"# HELP read_buffer_column_bytes The number of bytes used by all columns in the Read Buffer",
"# TYPE read_buffer_column_bytes gauge",
r#"read_buffer_column_bytes{db="mydb",encoding="BT_U32",log_data_type="i64"} 108"#,
r#"read_buffer_column_bytes{db="mydb",encoding="None",log_data_type="bool"} 1152"#,
r#"read_buffer_column_bytes{db="mydb",encoding="None",log_data_type="f64"} 1176"#,
r#"read_buffer_column_bytes{db="mydb",encoding="RLE",log_data_type="string"} 750"#,
r#"# HELP read_buffer_column_raw_bytes The number of bytes used by all columns if they were uncompressed in the Read Buffer"#,
r#"# TYPE read_buffer_column_raw_bytes gauge"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32",log_data_type="i64",null="false"} 144"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="bool",null="false"} 81"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="f64",null="false"} 264"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="f64",null="true"} 24"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="RLE",log_data_type="string",null="false"} 324"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="RLE",log_data_type="string",null="true"} 0"#,
r#"# HELP read_buffer_column_total The number of columns within the Read Buffer"#,
r#"# TYPE read_buffer_column_total gauge"#,
r#"read_buffer_column_total{db="mydb",encoding="BT_U32",log_data_type="i64"} 3"#,
r#"read_buffer_column_total{db="mydb",encoding="None",log_data_type="bool"} 3"#,
r#"read_buffer_column_total{db="mydb",encoding="None",log_data_type="f64"} 6"#,
r#"read_buffer_column_total{db="mydb",encoding="RLE",log_data_type="string"} 3"#,
r#"# HELP read_buffer_column_values The number of values within columns in the Read Buffer"#,
r#"# TYPE read_buffer_column_values gauge"#,
r#"read_buffer_column_values{db="mydb",encoding="BT_U32",log_data_type="i64",null="false"} 9"#,
r#"read_buffer_column_values{db="mydb",encoding="BT_U32",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="bool",null="false"} 9"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="f64",null="false"} 15"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="f64",null="true"} 3"#,
r#"read_buffer_column_values{db="mydb",encoding="RLE",log_data_type="string",null="false"} 9"#,
r#"read_buffer_column_values{db="mydb",encoding="RLE",log_data_type="string",null="true"} 0"#,
"",
"# HELP read_buffer_column_bytes The number of bytes used by all columns in the Read Buffer",
"# TYPE read_buffer_column_bytes gauge",
r#"read_buffer_column_bytes{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64"} 108"#,
r#"read_buffer_column_bytes{db="mydb",encoding="FIXED",log_data_type="f64"} 144"#,
r#"read_buffer_column_bytes{db="mydb",encoding="FIXEDN",log_data_type="bool"} 1152"#,
r#"read_buffer_column_bytes{db="mydb",encoding="FIXEDN",log_data_type="f64"} 1032"#,
r#"read_buffer_column_bytes{db="mydb",encoding="RLE",log_data_type="string"} 750"#,
"# HELP read_buffer_column_raw_bytes The number of bytes used by all columns if they were uncompressed in the Read Buffer",
"# TYPE read_buffer_column_raw_bytes gauge",
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="false"} 144"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXED",log_data_type="f64",null="false"} 144"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXED",log_data_type="f64",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="bool",null="false"} 81"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="f64",null="false"} 120"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="f64",null="true"} 24"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="RLE",log_data_type="string",null="false"} 324"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="RLE",log_data_type="string",null="true"} 0"#,
"# HELP read_buffer_column_total The number of columns within the Read Buffer",
"# TYPE read_buffer_column_total gauge",
r#"read_buffer_column_total{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64"} 3"#,
r#"read_buffer_column_total{db="mydb",encoding="FIXED",log_data_type="f64"} 3"#,
r#"read_buffer_column_total{db="mydb",encoding="FIXEDN",log_data_type="bool"} 3"#,
r#"read_buffer_column_total{db="mydb",encoding="FIXEDN",log_data_type="f64"} 3"#,
r#"read_buffer_column_total{db="mydb",encoding="RLE",log_data_type="string"} 3"#,
"# HELP read_buffer_column_values The number of values within columns in the Read Buffer",
"# TYPE read_buffer_column_values gauge",
r#"read_buffer_column_values{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="false"} 9"#,
r#"read_buffer_column_values{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXED",log_data_type="f64",null="false"} 9"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXED",log_data_type="f64",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="bool",null="false"} 9"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="f64",null="false"} 6"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="f64",null="true"} 3"#,
r#"read_buffer_column_values{db="mydb",encoding="RLE",log_data_type="string",null="false"} 9"#,
r#"read_buffer_column_values{db="mydb",encoding="RLE",log_data_type="string",null="true"} 0"#,
"",
]
.join("\n")
);
// when the chunk is dropped the metics are all correctly decreased
// when the chunk is dropped the metrics are all correctly decreased
std::mem::drop(chunk);
assert_eq!(
String::from_utf8(reg.registry().metrics_as_text()).unwrap(),
vec![
"# HELP read_buffer_column_bytes The number of bytes used by all columns in the Read Buffer",
"# TYPE read_buffer_column_bytes gauge",
r#"read_buffer_column_bytes{db="mydb",encoding="BT_U32",log_data_type="i64"} 0"#,
r#"read_buffer_column_bytes{db="mydb",encoding="None",log_data_type="bool"} 0"#,
r#"read_buffer_column_bytes{db="mydb",encoding="None",log_data_type="f64"} 0"#,
r#"read_buffer_column_bytes{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64"} 0"#,
r#"read_buffer_column_bytes{db="mydb",encoding="FIXED",log_data_type="f64"} 0"#,
r#"read_buffer_column_bytes{db="mydb",encoding="FIXEDN",log_data_type="bool"} 0"#,
r#"read_buffer_column_bytes{db="mydb",encoding="FIXEDN",log_data_type="f64"} 0"#,
r#"read_buffer_column_bytes{db="mydb",encoding="RLE",log_data_type="string"} 0"#,
r#"# HELP read_buffer_column_raw_bytes The number of bytes used by all columns if they were uncompressed in the Read Buffer"#,
r#"# TYPE read_buffer_column_raw_bytes gauge"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32",log_data_type="i64",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="bool",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="f64",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="None",log_data_type="f64",null="true"} 0"#,
"# HELP read_buffer_column_raw_bytes The number of bytes used by all columns if they were uncompressed in the Read Buffer",
"# TYPE read_buffer_column_raw_bytes gauge",
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXED",log_data_type="f64",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXED",log_data_type="f64",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="bool",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="f64",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="FIXEDN",log_data_type="f64",null="true"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="RLE",log_data_type="string",null="false"} 0"#,
r#"read_buffer_column_raw_bytes{db="mydb",encoding="RLE",log_data_type="string",null="true"} 0"#,
r#"# HELP read_buffer_column_total The number of columns within the Read Buffer"#,
r#"# TYPE read_buffer_column_total gauge"#,
r#"read_buffer_column_total{db="mydb",encoding="BT_U32",log_data_type="i64"} 0"#,
r#"read_buffer_column_total{db="mydb",encoding="None",log_data_type="bool"} 0"#,
r#"read_buffer_column_total{db="mydb",encoding="None",log_data_type="f64"} 0"#,
"# HELP read_buffer_column_total The number of columns within the Read Buffer",
"# TYPE read_buffer_column_total gauge",
r#"read_buffer_column_total{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64"} 0"#,
r#"read_buffer_column_total{db="mydb",encoding="FIXED",log_data_type="f64"} 0"#,
r#"read_buffer_column_total{db="mydb",encoding="FIXEDN",log_data_type="bool"} 0"#,
r#"read_buffer_column_total{db="mydb",encoding="FIXEDN",log_data_type="f64"} 0"#,
r#"read_buffer_column_total{db="mydb",encoding="RLE",log_data_type="string"} 0"#,
r#"# HELP read_buffer_column_values The number of values within columns in the Read Buffer"#,
r#"# TYPE read_buffer_column_values gauge"#,
r#"read_buffer_column_values{db="mydb",encoding="BT_U32",log_data_type="i64",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="BT_U32",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="bool",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="f64",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="None",log_data_type="f64",null="true"} 0"#,
"# HELP read_buffer_column_values The number of values within columns in the Read Buffer",
"# TYPE read_buffer_column_values gauge",
r#"read_buffer_column_values{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="BT_U32-FIXED",log_data_type="i64",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXED",log_data_type="f64",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXED",log_data_type="f64",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="bool",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="bool",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="f64",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="FIXEDN",log_data_type="f64",null="true"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="RLE",log_data_type="string",null="false"} 0"#,
r#"read_buffer_column_values{db="mydb",encoding="RLE",log_data_type="string",null="true"} 0"#,
"",

97
read_buffer/src/column.rs
View File

@ -339,7 +339,6 @@ impl Column {
EncodedValues::U32(dst) => EncodedValues::U32(data.encoded_values(row_ids, dst)),
typ => unimplemented!("column type String does not support {:?} encoding", typ),
},
Self::Integer(_, data) => data.encoded_values(row_ids, dst),
// Right now it only makes sense to expose encoded values on columns
// that are being used for grouping operations, which typically is
// are Tag Columns; they are `String` columns.
@ -354,7 +353,6 @@ impl Column {
EncodedValues::U32(dst) => EncodedValues::U32(data.all_encoded_values(dst)),
_ => unimplemented!("column type does not support requested encoding"),
},
Self::Integer(_, data) => data.all_encoded_values(dst),
// Right now it only makes sense to expose encoded values on columns
// that are being used for grouping operations, which typically is
// are Tag Columns (Strings) or Time Columns (Integers).
@ -1376,7 +1374,7 @@ impl Iterator for RowIDsIterator<'_> {
// Statistics about the composition of a column
pub(crate) struct Statistics {
pub enc_type: &'static str, // The encoding type
pub enc_type: String, // The encoding type
pub log_data_type: &'static str, // The logical data-type
pub values: u32, // Number of values present (NULL and non-NULL)
pub nulls: u32, // Number of NULL values present
@ -1517,49 +1515,13 @@ mod test {
let input = &[-1, i8::MAX as i64];
assert!(matches!(
Column::from(&input[..]),
Column::Integer(_, IntegerEncoding::I64I8(_))
));
let input = &[0, u8::MAX as i64];
assert!(matches!(
Column::from(&input[..]),
Column::Integer(_, IntegerEncoding::I64U8(_))
));
let input = &[-1, i16::MAX as i64];
assert!(matches!(
Column::from(&input[..]),
Column::Integer(_, IntegerEncoding::I64I16(_))
));
let input = &[0, u16::MAX as i64];
assert!(matches!(
Column::from(&input[..]),
Column::Integer(_, IntegerEncoding::I64U16(_))
));
let input = &[-1, i32::MAX as i64];
assert!(matches!(
Column::from(&input[..]),
Column::Integer(_, IntegerEncoding::I64I32(_))
));
let input = &[0, u32::MAX as i64];
assert!(matches!(
Column::from(&input[..]),
Column::Integer(_, IntegerEncoding::I64U32(_))
));
let input = &[-1, i64::MAX];
assert!(matches!(
Column::from(&input[..]),
Column::Integer(_, IntegerEncoding::I64I64(_))
Column::Integer(_, IntegerEncoding::I64(_, _))
));
// validate min/max check
let input = &[0, -12, u16::MAX as i64, 5];
let col = Column::from(&input[..]);
if let Column::Integer(meta, IntegerEncoding::I64I32(_)) = col {
if let Column::Integer(meta, IntegerEncoding::I64(_, _)) = col {
assert_eq!(meta.range, Some((-12, u16::MAX as i64)));
} else {
panic!("invalid variant");
@ -1571,25 +1533,13 @@ mod test {
let input = &[0, u8::MAX as u64];
assert!(matches!(
Column::from(&input[..]),
Column::Unsigned(_, IntegerEncoding::U64U8(_))
));
let input = &[0, u16::MAX as u64];
assert!(matches!(
Column::from(&input[..]),
Column::Unsigned(_, IntegerEncoding::U64U16(_))
));
let input = &[0, u32::MAX as u64];
assert!(matches!(
Column::from(&input[..]),
Column::Unsigned(_, IntegerEncoding::U64U32(_))
Column::Unsigned(_, IntegerEncoding::U64(_, _))
));
// validate min/max check
let input = &[13, 12, u16::MAX as u64, 5];
let col = Column::from(&input[..]);
if let Column::Unsigned(meta, IntegerEncoding::U64U16(_)) = col {
if let Column::Unsigned(meta, IntegerEncoding::U64(_, _)) = col {
assert_eq!(meta.range, Some((5, u16::MAX as u64)));
} else {
panic!("invalid variant");
@ -1742,19 +1692,6 @@ mod test {
} else {
panic!("not a EncodedValues::U32")
}
// timestamp column
let input = &[
1001231231243_i64,
1001231231343,
1001231231443,
1001231231543,
];
let col = Column::from(&input[..]);
assert_eq!(
col.encoded_values(&[0, 2], EncodedValues::I64(vec![])),
EncodedValues::I64(vec![1001231231243, 1001231231443])
);
}
#[test]
@ -1772,24 +1709,6 @@ mod test {
col.all_encoded_values(EncodedValues::U32(vec![])),
EncodedValues::U32(vec![1, NULL_ID, 2, 1, 2])
);
// timestamp column
let input = &[
1001231231243_i64,
1001231231343,
1001231231443,
1001231231543,
];
let col = Column::from(&input[..]);
assert_eq!(
col.all_encoded_values(EncodedValues::I64(vec![])),
EncodedValues::I64(vec![
1001231231243,
1001231231343,
1001231231443,
1001231231543,
])
);
}
#[test]
@ -2139,7 +2058,7 @@ mod test {
let col = Column::from(&input[..]);
assert!(matches!(
col,
Column::Integer(_, IntegerEncoding::I64I64(_))
Column::Integer(_, IntegerEncoding::I64(_, _))
));
let cases: Vec<(Scalar, bool)> = vec![
@ -2162,7 +2081,7 @@ mod test {
let col = Column::from(&input[..]);
assert!(matches!(
col,
Column::Integer(_, IntegerEncoding::I64U16(_))
Column::Integer(_, IntegerEncoding::I64(_, _))
));
for (scalar, result) in cases.clone() {
@ -2174,7 +2093,7 @@ mod test {
let col = Column::from(&input[..]);
assert!(matches!(
col,
Column::Unsigned(_, IntegerEncoding::U64U16(_))
Column::Unsigned(_, IntegerEncoding::U64(_, _))
));
for (scalar, result) in cases.clone() {

4
read_buffer/src/column/boolean.rs
View File

@ -36,7 +36,7 @@ impl BooleanEncoding {
// Returns statistics about the physical layout of columns
pub(crate) fn storage_stats(&self) -> Statistics {
Statistics {
enc_type: self.name(),
enc_type: self.name().to_string(),
log_data_type: "bool",
values: self.num_rows(),
nulls: self.null_count(),
@ -141,7 +141,7 @@ impl BooleanEncoding {
/// The name of this encoding.
pub fn name(&self) -> &'static str {
match &self {
Self::BooleanNull(_) => "None",
Self::BooleanNull(_) => "FIXEDN",
}
}
}

4
read_buffer/src/column/string.rs
View File

@ -73,8 +73,8 @@ impl StringEncoding {
pub(crate) fn storage_stats(&self) -> Statistics {
Statistics {
enc_type: match self {
Self::RleDictionary(_) => rle::ENCODING_NAME,
Self::Dictionary(_) => dictionary::ENCODING_NAME,
Self::RleDictionary(_) => rle::ENCODING_NAME.to_string(),
Self::Dictionary(_) => dictionary::ENCODING_NAME.to_string(),
},
log_data_type: "string",
values: self.num_rows(),