From bbfff8699c010a8496f3797a253826936b00a2c0 Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" Date: Mon, 27 Feb 2023 11:57:44 -0500 Subject: [PATCH] fix: Use the same normalization code for explain tests as e2e tests do The regex for replacing UUIDs needed to be changed like the normalizer's regex did, so keep them in sync by using the same code. This might point to the normalizer needing to be moved somewhere else, or changing these tests to be e2e? --- Cargo.lock | 2 +- querier/Cargo.toml | 2 +- querier/src/namespace/query_access.rs | 88 +++++++++---------- test_helpers_end_to_end/src/lib.rs | 2 +- .../src/snapshot_comparison.rs | 2 +- 5 files changed, 46 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e383b52036..8e4eb90655 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4513,7 +4513,6 @@ dependencies = [ "predicate", "prost", "rand", - "regex", "schema", "service_common", "service_grpc_catalog", @@ -4522,6 +4521,7 @@ dependencies = [ "sharder", "snafu", "test_helpers", + "test_helpers_end_to_end", "thiserror", "tokio", "tokio-util", diff --git a/querier/Cargo.toml b/querier/Cargo.toml index 5bdb8d69c1..90689d93d1 100644 --- a/querier/Cargo.toml +++ b/querier/Cargo.toml @@ -56,5 +56,5 @@ insta = { version = "1.28.0", features = ["yaml"] } iox_tests = { path = "../iox_tests" } mutable_batch_lp = { path = "../mutable_batch_lp" } object_store_metrics = { path = "../object_store_metrics" } -regex = "1.7.1" test_helpers = { path = "../test_helpers" } +test_helpers_end_to_end = { path = "../test_helpers_end_to_end" } diff --git a/querier/src/namespace/query_access.rs b/querier/src/namespace/query_access.rs index 36a11f0f5c..e7221686a5 100644 --- a/querier/src/namespace/query_access.rs +++ b/querier/src/namespace/query_access.rs @@ -205,8 +205,8 @@ mod tests { use iox_query::frontend::sql::SqlQueryPlanner; use iox_tests::{TestCatalog, TestParquetFileBuilder}; use metric::{Observation, RawReporter}; - use regex::Regex; use snafu::{ResultExt, Snafu}; + use test_helpers_end_to_end::snapshot_comparison::normalization::Normalizer; use trace::{span::SpanStatus, RingBufferTraceCollector}; #[tokio::test] @@ -492,13 +492,13 @@ mod tests { format_explain(&querier_namespace, "EXPLAIN SELECT * FROM cpu").await, @r###" --- - - +---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - - "| plan_type | plan |" - - +---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - - "| logical_plan | TableScan: cpu projection=[foo, host, load, time] |" - - "| physical_plan | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/.parquet, 1/1/1/1/.parquet, 1/1/1/1/.parquet, 1/1/2/2/.parquet, 1/1/1/3/.parquet]]}, projection=[foo, host, load, time] |" - - "| | |" - - +---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + - "----------" + - "| plan_type | plan |" + - "----------" + - "| logical_plan | TableScan: cpu projection=[foo, host, load, time] |" + - "| physical_plan | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/00000000-0000-0000-0000-000000000000.parquet, 1/1/1/1/00000000-0000-0000-0000-000000000001.parquet, 1/1/1/1/00000000-0000-0000-0000-000000000002.parquet, 1/1/1/1/00000000-0000-0000-0000-000000000003.parquet, 1/1/1/1/00000000-0000-0000-0000-000000000004.parquet]]}, projection=[foo, host, load, time] |" + - "| | |" + - "----------" "### ); @@ -509,22 +509,22 @@ mod tests { format_explain(&querier_namespace, "EXPLAIN SELECT * FROM mem ORDER BY host,time").await, @r###" --- - - +---------------+--------------------------------------------------------------------------------------------------------------------------------------------------+ - - "| plan_type | plan |" - - +---------------+--------------------------------------------------------------------------------------------------------------------------------------------------+ - - "| logical_plan | Sort: mem.host ASC NULLS LAST, mem.time ASC NULLS LAST |" - - "| | TableScan: mem projection=[host, perc, time] |" - - "| physical_plan | SortExec: expr=[host@0 ASC NULLS LAST,time@2 ASC NULLS LAST] |" - - "| | CoalescePartitionsExec |" - - "| | UnionExec |" - - "| | CoalesceBatchesExec: target_batch_size=8192 |" - - "| | FilterExec: time@2 < 1 OR time@2 > 13 OR NOT host@0 = CAST(d AS Dictionary(Int32, Utf8)) |" - - "| | ParquetExec: limit=None, partitions={1 group: [[1/2/1/4/.parquet]]}, projection=[host, perc, time] |" - - "| | CoalesceBatchesExec: target_batch_size=8192 |" - - "| | FilterExec: time@2 < 1 OR time@2 > 13 OR NOT host@0 = CAST(d AS Dictionary(Int32, Utf8)) |" - - "| | ParquetExec: limit=None, partitions={1 group: [[1/2/1/4/.parquet]]}, projection=[host, perc, time] |" - - "| | |" - - +---------------+--------------------------------------------------------------------------------------------------------------------------------------------------+ + - "----------" + - "| plan_type | plan |" + - "----------" + - "| logical_plan | Sort: mem.host ASC NULLS LAST, mem.time ASC NULLS LAST |" + - "| | TableScan: mem projection=[host, perc, time] |" + - "| physical_plan | SortExec: expr=[host@0 ASC NULLS LAST,time@2 ASC NULLS LAST] |" + - "| | CoalescePartitionsExec |" + - "| | UnionExec |" + - "| | CoalesceBatchesExec: target_batch_size=8192 |" + - "| | FilterExec: time@2 < 1 OR time@2 > 13 OR NOT host@0 = CAST(d AS Dictionary(Int32, Utf8)) |" + - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/00000000-0000-0000-0000-000000000000.parquet]]}, projection=[host, perc, time] |" + - "| | CoalesceBatchesExec: target_batch_size=8192 |" + - "| | FilterExec: time@2 < 1 OR time@2 > 13 OR NOT host@0 = CAST(d AS Dictionary(Int32, Utf8)) |" + - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/00000000-0000-0000-0000-000000000001.parquet]]}, projection=[host, perc, time] |" + - "| | |" + - "----------" "### ); @@ -567,19 +567,19 @@ mod tests { format_explain(&querier_namespace, "EXPLAIN SELECT * FROM cpu").await, @r###" --- - - +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - - "| plan_type | plan |" - - +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - - "| logical_plan | TableScan: cpu projection=[foo, host, load, time] |" - - "| physical_plan | UnionExec |" - - "| | DeduplicateExec: [host@1 ASC,time@3 ASC] |" - - "| | SortPreservingMergeExec: [host@1 ASC,time@3 ASC] |" - - "| | UnionExec |" - - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/2/2/.parquet]]}, output_ordering=[host@1 ASC, time@3 ASC], projection=[foo, host, load, time] |" - - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/2/2/.parquet]]}, output_ordering=[host@1 ASC, time@3 ASC], projection=[foo, host, load, time] |" - - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/.parquet, 1/1/1/1/.parquet, 1/1/1/1/.parquet, 1/1/1/3/.parquet]]}, projection=[foo, host, load, time] |" - - "| | |" - - +---------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + - "----------" + - "| plan_type | plan |" + - "----------" + - "| logical_plan | TableScan: cpu projection=[foo, host, load, time] |" + - "| physical_plan | UnionExec |" + - "| | DeduplicateExec: [host@1 ASC,time@3 ASC] |" + - "| | SortPreservingMergeExec: [host@1 ASC,time@3 ASC] |" + - "| | UnionExec |" + - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/00000000-0000-0000-0000-000000000000.parquet]]}, output_ordering=[host@1 ASC, time@3 ASC], projection=[foo, host, load, time] |" + - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/00000000-0000-0000-0000-000000000001.parquet]]}, output_ordering=[host@1 ASC, time@3 ASC], projection=[foo, host, load, time] |" + - "| | ParquetExec: limit=None, partitions={1 group: [[1/1/1/1/00000000-0000-0000-0000-000000000002.parquet, 1/1/1/1/00000000-0000-0000-0000-000000000003.parquet, 1/1/1/1/00000000-0000-0000-0000-000000000004.parquet, 1/1/1/1/00000000-0000-0000-0000-000000000005.parquet]]}, projection=[foo, host, load, time] |" + - "| | |" + - "----------" "### ); } @@ -599,15 +599,11 @@ mod tests { async fn format_explain(querier_namespace: &Arc, sql: &str) -> Vec { let results = run(querier_namespace, sql, None).await; - let formatted = arrow_util::display::pretty_format_batches(&results).unwrap(); - - let regex = Regex::new("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") - .expect("UUID regex"); - formatted - .trim() - .split('\n') - .map(|s| regex.replace_all(s, "").to_string()) - .collect::>() + let normalizer = Normalizer { + normalized_uuids: true, + ..Default::default() + }; + normalizer.normalize_results(results) } async fn run( diff --git a/test_helpers_end_to_end/src/lib.rs b/test_helpers_end_to_end/src/lib.rs index 152e4124e7..6e58e27f64 100644 --- a/test_helpers_end_to_end/src/lib.rs +++ b/test_helpers_end_to_end/src/lib.rs @@ -13,7 +13,7 @@ mod grpc; mod mini_cluster; mod server_fixture; mod server_type; -mod snapshot_comparison; +pub mod snapshot_comparison; mod steps; mod udp_listener; diff --git a/test_helpers_end_to_end/src/snapshot_comparison.rs b/test_helpers_end_to_end/src/snapshot_comparison.rs index 118b33db54..4ec43d2c6f 100644 --- a/test_helpers_end_to_end/src/snapshot_comparison.rs +++ b/test_helpers_end_to_end/src/snapshot_comparison.rs @@ -1,4 +1,4 @@ -mod normalization; +pub mod normalization; mod queries; use crate::snapshot_comparison::queries::TestQueries;