//! The setups available for any `TestCase` to use by specifying the test name in a comment at the //! start of the `.sql` file in the form of: //! //! ```text //! -- IOX_SETUP: [test name] //! ``` use iox_time::{SystemProvider, Time, TimeProvider}; use once_cell::sync::Lazy; use std::collections::HashMap; use test_helpers_end_to_end::Step; /// The string value that will appear in `.sql` files. pub type SetupName = &'static str; /// The steps that should be run when this setup is chosen. pub type SetupSteps = Vec; /// timestamps for the retention test static RETENTION_SETUP: Lazy = Lazy::new(RetentionSetup::new); /// All possible setups for the [`TestCase`][crate::TestCase]s to use, indexed by name pub static SETUPS: Lazy> = Lazy::new(|| { HashMap::from([ ( "Bugs", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=66909i,status=204i 1678578528255989730"#, r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=112928i,status=204i 1678578532192972168"#, r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=147683i,status=204i 1678578588416052133"#, r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=78000i,status=204i 1678578592147893402"#, ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurements", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "cpu,region=west user=23.2 100", "cpu,region=west user=21.0 150", "disk,region=east bytes=99i 200", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, ], ), ( "TwoChunksDedupWeirdnessParquet", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol("table,tag=A foo=1,bar=1 0".into()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol(["table,tag=A bar=2 0", "table,tag=B foo=1 0"].join("\n")), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoChunksDedupWeirdnessParquetIngester", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol("table,tag=A foo=1,bar=1 0".into()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::WriteLineProtocol(["table,tag=A bar=2 0", "table,tag=B foo=1 0"].join("\n")), ], ), ( "OneMeasurementFourChunksWithDuplicatesWithIngester", vec![ Step::RecordNumParquetFiles, // Chunk 1: // . time range: 50-250 // . no duplicates in its own chunk Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston min_temp=70.4 50", "h2o,state=MA,city=Bedford min_temp=71.59 150", "h2o,state=MA,city=Boston max_temp=75.4 250", "h2o,state=MA,city=Andover max_temp=69.2, 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // Chunk 2: overlaps with chunk 1 // . time range: 150 - 300 // . no duplicates in its own chunk Step::WriteLineProtocol( [ // new field (area) and update available NULL (max_temp) "h2o,state=MA,city=Bedford max_temp=78.75,area=742u 150", "h2o,state=MA,city=Boston min_temp=65.4 250", // update min_temp from NULL "h2o,state=MA,city=Reading min_temp=53.4, 250", "h2o,state=CA,city=SF min_temp=79.0,max_temp=87.2,area=500u 300", "h2o,state=CA,city=SJ min_temp=78.5,max_temp=88.0 300", "h2o,state=CA,city=SJ min_temp=75.5,max_temp=84.08 350", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // Chunk 3: no overlap // . time range: 400 - 500 // . duplicates in its own chunk Step::WriteLineProtocol( [ "h2o,state=MA,city=Bedford max_temp=80.75,area=742u 400", "h2o,state=MA,city=Boston min_temp=68.4 400", "h2o,state=MA,city=Bedford min_temp=65.22,area=750u 400", // duplicate "h2o,state=MA,city=Boston min_temp=65.40,max_temp=82.67 400", // duplicate "h2o,state=CA,city=SJ min_temp=77.0,max_temp=90.7 450", "h2o,state=CA,city=SJ min_temp=69.5,max_temp=88.2 500", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, // Chunk 4: no overlap // . time range: 600 - 700 // . no duplicates Step::WriteLineProtocol( [ "h2o,state=MA,city=Bedford max_temp=88.75,area=742u 600", "h2o,state=MA,city=Boston min_temp=67.4 600", "h2o,state=MA,city=Reading min_temp=60.4, 600", "h2o,state=CA,city=SF min_temp=68.4,max_temp=85.7,area=500u 650", "h2o,state=CA,city=SJ min_temp=69.5,max_temp=89.2 650", "h2o,state=CA,city=SJ min_temp=75.5,max_temp=84.08 700", ] .join("\n"), ), ], ), ( "OneMeasurementFourChunksWithDuplicatesParquetOnly", vec![ Step::RecordNumParquetFiles, // Chunk 1: // . time range: 50-250 // . no duplicates in its own chunk Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston min_temp=70.4 50", "h2o,state=MA,city=Bedford min_temp=71.59 150", "h2o,state=MA,city=Boston max_temp=75.4 250", "h2o,state=MA,city=Andover max_temp=69.2, 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // Chunk 2: overlaps with chunk 1 // . time range: 150 - 300 // . no duplicates in its own chunk Step::WriteLineProtocol( [ // new field (area) and update available NULL (max_temp) "h2o,state=MA,city=Bedford max_temp=78.75,area=742u 150", "h2o,state=MA,city=Boston min_temp=65.4 250", // update min_temp from NULL "h2o,state=MA,city=Reading min_temp=53.4, 250", "h2o,state=CA,city=SF min_temp=79.0,max_temp=87.2,area=500u 300", "h2o,state=CA,city=SJ min_temp=78.5,max_temp=88.0 300", "h2o,state=CA,city=SJ min_temp=75.5,max_temp=84.08 350", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // Chunk 3: no overlap // . time range: 400 - 500 // . duplicates in its own chunk Step::WriteLineProtocol( [ "h2o,state=MA,city=Bedford max_temp=80.75,area=742u 400", "h2o,state=MA,city=Boston min_temp=68.4 400", "h2o,state=MA,city=Bedford min_temp=65.22,area=750u 400", // duplicate "h2o,state=MA,city=Boston min_temp=65.40,max_temp=82.67 400", // duplicate "h2o,state=CA,city=SJ min_temp=77.0,max_temp=90.7 450", "h2o,state=CA,city=SJ min_temp=69.5,max_temp=88.2 500", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // Chunk 4: no overlap // . time range: 600 - 700 // . no duplicates Step::WriteLineProtocol( [ "h2o,state=MA,city=Bedford max_temp=88.75,area=742u 600", "h2o,state=MA,city=Boston min_temp=67.4 600", "h2o,state=MA,city=Reading min_temp=60.4, 600", "h2o,state=CA,city=SF min_temp=68.4,max_temp=85.7,area=500u 650", "h2o,state=CA,city=SJ min_temp=69.5,max_temp=89.2 650", "h2o,state=CA,city=SJ min_temp=75.5,max_temp=84.08 700", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwentySortedParquetFiles", (0..20) .flat_map(|i| { let write = if i % 2 == 0 { Step::WriteLineProtocol(format!( "m,tag=A f=1 {}\nm,tab=B f=2 {}", 1000 - i, // unique in this chunk 1000 - i, // unique in this chunk (not plus i!) )) } else { Step::WriteLineProtocol( "m,tag=A f=3 2001".into(), // duplicated across all chunks ) }; [ Step::RecordNumParquetFiles, write, Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ] .into_iter() }) .collect::>(), ), ( "TwentySortedParquetFilesAndIngester", (0..20) .flat_map(|i| { let write = if i % 2 == 0 { Step::WriteLineProtocol(format!( "m,tag=A f=1 {}\nm,tab=B f=2 {}", 1000 - i, // unique in this chunk 1000 - i, // unique in this chunk (not plus i!) )) } else { Step::WriteLineProtocol( "m,tag=A f=3 2001".into(), // duplicated across all chunks ) }; [ Step::RecordNumParquetFiles, write, Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ] .into_iter() }).chain([ Step::WriteLineProtocol( "m,tag=A f=3 2001".into(), // duplicated across all chunks ) ]) .collect::>(), ), ( "FiftySortedSameParquetFiles", (0..50) .flat_map(|_i| { let write = Step::WriteLineProtocol( "m,tag1=A,tag2=B,tag3=C,tag4=D f1=1,f2=2 2001".into(), // duplicated across all chunks ); [ Step::RecordNumParquetFiles, write, Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ] .into_iter() }) .collect::>(), ), ( "OneMeasurementManyFields", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,tag1=foo,tag2=bar field1=70.6,field3=2 100", "h2o,tag1=foo,tag2=bar field1=70.4,field2=\"ss\" 100", "h2o,tag1=foo,tag2=bar field1=70.5,field2=\"ss\" 100", "h2o,tag1=foo,tag2=bar field1=70.6,field4=true 1000", "h2o,tag1=foo,tag2=bar field1=70.3,field5=false 3000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurementsManyFields", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.4 50", "h2o,state=MA,city=Boston other_temp=70.4 250", "h2o,state=CA,city=Boston other_temp=72.4 350", "o2,state=MA,city=Boston temp=53.4,reading=51 50", "o2,state=CA temp=79.0 300", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( "h2o,state=MA,city=Boston temp=70.4,moisture=43.0 100000".into(), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurementsManyFieldsTwoChunks", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.4 50", "h2o,state=MA,city=Boston other_temp=70.4 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::WriteLineProtocol( [ "h2o,state=CA,city=Boston other_temp=72.4 150", "o2,state=MA,city=Boston temp=53.4,reading=51 50", "o2,state=CA temp=79.0 300", ] .join("\n"), ), // The system tables test looks for queries, so the setup needs to run this query. Step::Query { sql: "SELECT 1;".into(), expected: vec![ "+----------+", "| Int64(1) |", "+----------+", "| 1 |", "+----------+", ], }, ], ), ( "TwoMeasurementsManyNulls", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=CA,city=LA,county=LA temp=70.4 100", "h2o,state=MA,city=Boston,county=Suffolk temp=72.4 250", "o2,state=MA,city=Boston temp=50.4 200", "o2,state=CA temp=79.0 300", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "o2,state=NY temp=60.8 400", "o2,state=NY,city=NYC temp=61.0 500", "o2,state=NY,city=NYC,borough=Brooklyn temp=61.0 600", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "OneMeasurementWithTags", vec![Step::WriteLineProtocol( [ "cpu,foo=me bar=1 10", "cpu,foo=you bar=2 20", "cpu,foo=me bar=1 30", "cpu,foo=me bar=1 40", ] .join("\n"), )], ), ( "EqualInMeasurements", vec![Step::WriteLineProtocol( [ "measurement=one,tag=value field=1.0 \ 1609459201000000001", "measurement=one,tag=value2 field=1.0 \ 1609459201000000002", ] .join("\n"), )], ), ( "PeriodsInNames", vec![Step::WriteLineProtocol( [ "measurement.one,tag.one=value,tag.two=other field.one=1.0,field.two=t \ 1609459201000000001", "measurement.one,tag.one=value2,tag.two=other2 field.one=1.0,field.two=f \ 1609459201000000002", ] .join("\n"), )], ), ( "TwoMeasurementsPredicatePushDown", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "restaurant,town=andover count=40000u,system=5.0 100", "restaurant,town=reading count=632u,system=5.0 120", "restaurant,town=bedford count=189u,system=7.0 110", "restaurant,town=tewsbury count=471u,system=6.0 110", "restaurant,town=lexington count=372u,system=5.0 100", "restaurant,town=lawrence count=872u,system=6.0 110", "restaurant,town=reading count=632u,system=6.0 130", ] .join("\n"), ), Step::WriteLineProtocol( [ "school,town=reading count=17u,system=6.0 150", "school,town=andover count=25u,system=6.0 160", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, ], ), ( "AllTypes", vec![Step::WriteLineProtocol( [ "m,tag=row1 float_field=64.0 450", "m,tag=row1 int_field=64 550", "m,tag=row1 \ float_field=61.0,int_field=22,uint_field=25u,\ string_field=\"foo\",bool_field=t 500", "m,tag=row1 \ float_field=62.0,int_field=21,uint_field=30u,\ string_field=\"ba\",bool_field=f 200", "m,tag=row1 \ float_field=63.0,int_field=20,uint_field=35u,\ string_field=\"baz\",bool_field=f 300", "m,tag=row1 \ float_field=64.0,int_field=19,uint_field=20u,\ string_field=\"bar\",bool_field=t 400", "m,tag=row1 \ float_field=65.0,int_field=18,uint_field=40u,\ string_field=\"fruz\",bool_field=f 100", "m,tag=row1 \ float_field=66.0,int_field=17,uint_field=10u,\ string_field=\"faa\",bool_field=t 600", ] .join("\n"), )], ), ( "ManyFieldsSeveralChunks", vec![ Step::RecordNumParquetFiles, // c1: parquet stage Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.4 50", // duplicate with a row in c4 and will be removed "h2o,state=MA,city=Boston other_temp=70.4 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // c2: parquet stage & overlaps with c1 Step::WriteLineProtocol("h2o,state=CA,city=Andover other_temp=72.4 150".into()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // c3: parquet stage & doesn't overlap with any Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=80.7 350", "h2o,state=MA,city=Boston other_temp=68.2 450", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, // c4: parquet stage & overlap with c1 Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=88.6 230", // duplicate with a row in c1 but more // recent => this row is kept "h2o,state=MA,city=Boston other_temp=80 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, // c5: ingester stage & doesn't overlap with any Step::WriteLineProtocol("h2o,state=CA,city=Andover temp=67.3 500".into()), ], ), ( "MeasurementWithMaxTime", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol(format!( "cpu,host=server01 value=100 {}", // This is the maximum timestamp that can be represented in the InfluxDB data // model: // i64::MAX - 1, // 9223372036854775806 )), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "OneMeasurementRealisticTimes", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "cpu,region=west user=23.2 1626809330000000000", "cpu,region=west user=21.0 1626809430000000000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "OneMeasurementTwoSeries", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "cpu,region=a user=23.2,idle=70.0 957529200000000000", // 2000-05-05T12:20:00Z "cpu,region=b user=25.2 957529860000000000", // 2000-05-05T12:31:00Z "cpu,region=b user=28.9,idle=60.0 957530340000000000", // 2000-05-05T12:39:00Z "cpu,region=a user=21.0 957530400000000000", // 2000-05-05T12:40:00Z ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoChunksMissingColumns", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol("table,tag1=a,tag2=b field1=10,field2=11 100".into()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol("table,tag1=a,tag3=c field1=20,field3=22 200".into()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurementsMultiSeries", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ // Data is deliberately not in series order. "h2o,state=CA,city=LA temp=90.0 200", "h2o,state=MA,city=Boston temp=72.4 250", "h2o,state=MA,city=Boston temp=70.4 100", "h2o,state=CA,city=LA temp=90.0 350", "o2,state=MA,city=Boston temp=53.4,reading=51 250", "o2,state=MA,city=Boston temp=50.4,reading=50 100", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, ], ), ( // This recreates the test case for . "StringFieldWithNumericValue", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( ["m,tag0=foo fld=\"200\" 1000", "m,tag0=foo fld=\"404\" 1050"].join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "MeasurementStatusCode", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "status_code,url=http://www.example.com value=404 1527018806000000000", "status_code,url=https://influxdb.com value=418 1527018816000000000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurementsMultiTagValue", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.4 100", "h2o,state=MA,city=Lowell temp=75.4 100", "h2o,state=CA,city=LA temp=90.0 200", "o2,state=MA,city=Boston temp=50.4,reading=50 100", "o2,state=KS,city=Topeka temp=60.4,reading=60 100", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, ], ), ( "MeasurementsSortableTags", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,zz_tag=A,state=MA,city=Kingston temp=70.1 800", "h2o,state=MA,city=Kingston,zz_tag=B temp=70.2 100", "h2o,state=CA,city=Boston temp=70.3 250", "h2o,state=MA,city=Boston,zz_tag=A temp=70.4 1000", "h2o,state=MA,city=Boston temp=70.5,other=5.0 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( // See issue: https://github.com/influxdata/influxdb_iox/issues/2845 "MeasurementsForDefect2845", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "system,host=host.local load1=1.83 1527018806000000000", "system,host=host.local load1=1.63 1527018816000000000", "system,host=host.local load3=1.72 1527018806000000000", "system,host=host.local load4=1.77 1527018806000000000", "system,host=host.local load4=1.78 1527018816000000000", "system,host=host.local load4=1.77 1527018826000000000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "EndToEndTest", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "cpu_load_short,host=server01,region=us-west value=0.64 0000", "cpu_load_short,host=server01 value=27.99 1000", "cpu_load_short,host=server02,region=us-west value=3.89 2000", "cpu_load_short,host=server01,region=us-east value=1234567.891011 3000", "cpu_load_short,host=server01,region=us-west value=0.000003 4000", "system,host=server03 uptime=1303385 5000", "swap,host=server01,name=disk0 in=3,out=4 6000", "status active=t 7000", "attributes color=\"blue\" 8000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 5, }, ], ), ( "OneMeasurementNoTags", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o temp=70.4 100", "h2o temp=72.4 250", "h2o temp=50.4 200", "h2o level=200.0 300", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "OneMeasurementNoTags2", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol(["m0 foo=1.0 1", "m0 foo=2.0 2"].join("\n")), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "OneMeasurementForAggs", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.4 100", "h2o,state=MA,city=Boston temp=72.4 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=CA,city=LA temp=90.0 200", "h2o,state=CA,city=LA temp=90.0 350", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurementForAggs", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.4 100", "h2o,state=MA,city=Boston temp=72.4 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "o2,state=CA,city=LA temp=90.0 200", "o2,state=CA,city=LA temp=90.0 350", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "AnotherMeasurementForAggs", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Cambridge temp=80 50", "h2o,state=MA,city=Cambridge temp=81 100", "h2o,state=MA,city=Cambridge temp=82 200", "h2o,state=MA,city=Boston temp=70 300", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=71 400", "h2o,state=CA,city=LA temp=90,humidity=10 500", "h2o,state=CA,city=LA temp=91,humidity=11 600", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "MeasurementForSelectors", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( ["h2o,state=MA,city=Cambridge f=8.0,i=8i,b=true,s=\"d\" 1000"].join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Cambridge f=7.0,i=7i,b=true,s=\"c\" 2000", "h2o,state=MA,city=Cambridge f=6.0,i=6i,b=false,s=\"b\" 3000", "h2o,state=MA,city=Cambridge f=5.0,i=5i,b=false,s=\"a\" 4000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "MeasurementForMin", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Cambridge f=8.0,i=8i,b=false,s=\"c\" 1000", "h2o,state=MA,city=Cambridge f=7.0,i=7i,b=true,s=\"a\" 2000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Cambridge f=6.0,i=6i,b=true,s=\"z\" 3000", "h2o,state=MA,city=Cambridge f=5.0,i=5i,b=false,s=\"c\" 4000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "MeasurementForMax", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Cambridge f=8.0,i=8i,b=true,s=\"c\" 1000", "h2o,state=MA,city=Cambridge f=7.0,i=7i,b=false,s=\"d\" 2000", "h2o,state=MA,city=Cambridge f=6.0,i=6i,b=true,s=\"a\" 3000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( ["h2o,state=MA,city=Cambridge f=5.0,i=5i,b=false,s=\"z\" 4000"].join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "MeasurementForGroupKeys", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Cambridge temp=80 50", "h2o,state=MA,city=Cambridge temp=81 100", "h2o,state=MA,city=Cambridge temp=82 200", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70 300", "h2o,state=MA,city=Boston temp=71 400", "h2o,state=CA,city=LA temp=90,humidity=10 500", "h2o,state=CA,city=LA temp=91,humidity=11 600", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "MeasurementForGroupByField", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "system,host=local,region=A load1=1.1,load2=2.1 100", "system,host=local,region=A load1=1.2,load2=2.2 200", "system,host=remote,region=B load1=10.1,load2=2.1 100", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "system,host=remote,region=B load1=10.2,load2=20.2 200", "system,host=local,region=C load1=100.1,load2=200.1 100", "aa_system,host=local,region=C load1=100.1,load2=200.1 100", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurementsManyFieldsOneChunk", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.4 50", "h2o,state=MA,city=Boston other_temp=70.4 250", "h2o,state=CA,city=Boston other_temp=72.4 350", "o2,state=MA,city=Boston temp=53.4,reading=51 50", "o2,state=CA temp=79.0 300", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, ], ), ( "MeasurementForDefect2691", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "system,host=host.local load1=1.83 1527018806000000000", "system,host=host.local load1=1.63 1527018816000000000", "system,host=host.local load3=1.72 1527018806000000000", "system,host=host.local load4=1.77 1527018806000000000", "system,host=host.local load4=1.78 1527018816000000000", "system,host=host.local load4=1.77 1527018826000000000", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "MeasurementForWindowAggregate", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Boston temp=70.0 100", "h2o,state=MA,city=Boston temp=71.0 200", "h2o,state=MA,city=Boston temp=72.0 300", "h2o,state=MA,city=Boston temp=73.0 400", "h2o,state=MA,city=Boston temp=74.0 500", "h2o,state=MA,city=Cambridge temp=80.0 100", "h2o,state=MA,city=Cambridge temp=81.0 200", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "h2o,state=MA,city=Cambridge temp=82.0 300", "h2o,state=MA,city=Cambridge temp=83.0 400", "h2o,state=MA,city=Cambridge temp=84.0 500", "h2o,state=CA,city=LA temp=90.0 100", "h2o,state=CA,city=LA temp=91.0 200", "h2o,state=CA,city=LA temp=92.0 300", "h2o,state=CA,city=LA temp=93.0 400", "h2o,state=CA,city=LA temp=94.0 500", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( // Test data to validate fix for // "MeasurementForDefect2697", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "mm,section=1a bar=5.0 1609459201000000011", "mm,section=1a bar=0.28 1609459201000000031", "mm,section=2b bar=4.0 1609459201000000009", "mm,section=2b bar=6.0 1609459201000000015", "mm,section=2b bar=1.2 1609459201000000022", "mm,section=1a foo=1.0 1609459201000000001", "mm,section=1a foo=3.0 1609459201000000005", "mm,section=1a foo=11.24 1609459201000000024", "mm,section=2b foo=2.0 1609459201000000002", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "ThreeChunksWithRetention", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol(RETENTION_SETUP.lp_partially_inside_1.clone()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol(RETENTION_SETUP.lp_fully_inside.clone()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol(RETENTION_SETUP.lp_fully_outside.clone()), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::WriteLineProtocol(RETENTION_SETUP.lp_partially_inside_2.clone()), Step::SetRetention(Some(RETENTION_SETUP.retention_period_ns)), ], ), ( // Test data to validate fix for // "MeasurementForDefect2890", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "mm foo=2.0 1609459201000000001", "mm foo=2.0 1609459201000000002", "mm foo=3.0 1609459201000000005", "mm foo=11.24 1609459201000000024", "mm bar=4.0 1609459201000000009", "mm bar=5.0 1609459201000000011", "mm bar=6.0 1609459201000000015", "mm bar=1.2 1609459201000000022", "mm bar=2.8 1609459201000000031", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( // Single measurement that has several different chunks with different (but // compatible) schemas "MultiChunkSchemaMerge", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "cpu,region=west user=23.2,system=5.0 100", "cpu,region=west user=21.0,system=6.0 150", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "cpu,region=east,host=foo user=23.2 100", "cpu,region=west,host=bar user=21.0 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( "TwoMeasurementsUnsignedType", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ "restaurant,town=andover count=40000u 100", "restaurant,town=reading count=632u 120", "school,town=reading count=17u 150", "school,town=andover count=25u 160", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( // This has two chunks with different tag/key sets for queries whose columns do not // include keys "OneMeasurementTwoChunksDifferentTagSet", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ // tag: state "h2o,state=MA temp=70.4 50", "h2o,state=MA other_temp=70.4 250", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( [ // tag: city "h2o,city=Boston other_temp=72.4 350", "h2o,city=Boston temp=53.4,reading=51 50", ] .join("\n"), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ], ), ( // This is the dataset defined by https://github.com/influxdata/influxdb_iox/issues/6112 "InfluxQLSelectSupport", vec![ Step::RecordNumParquetFiles, Step::WriteLineProtocol( r#" m0,tag0=val00 f64=10.1,i64=101i,str="hi" 1667181600000000000 m0,tag0=val00 f64=21.2,i64=211i,str="hi" 1667181610000000000 m0,tag0=val00 f64=11.2,i64=191i,str="lo" 1667181620000000000 m0,tag0=val00 f64=19.2,i64=392i,str="lo" 1667181630000000000 m0,tag0=val01 f64=11.3,i64=211i,str="lo" 1667181600000000000 m0,tag0=val02 f64=10.4,i64=101i,str="lo" 1667181600000000000 m0,tag0=val00,tag1=val10 f64=18.9,i64=211i,str="lo" 1667181610000000000 m1,tag0=val00 f64=100.5,i64=1001i,str="hi" 1667181600000000000 m1,tag0=val00 f64=200.6,i64=2001i,str="lo" 1667181610000000000 m1,tag0=val01 f64=101.7,i64=1011i,str="lo" 1667181600000000000 cpu,host=host1,cpu=cpu-total usage_idle=2.98,usage_system=2.2 1667181600000000000 cpu,host=host1,cpu=cpu-total usage_idle=2.99,usage_system=2.1 1667181610000000000 cpu,host=host1,cpu=cpu0 usage_idle=0.98,usage_system=0.2 1667181600000000000 cpu,host=host1,cpu=cpu0 usage_idle=0.99,usage_system=0.1 1667181610000000000 cpu,host=host1,cpu=cpu1 usage_idle=1.98,usage_system=1.2 1667181600000000000 cpu,host=host1,cpu=cpu1 usage_idle=1.99,usage_system=1.1 1667181610000000000 disk,host=host1,device=disk1s1 bytes_free=1234i,bytes_used=219838i 1667181600000000000 disk,host=host1,device=disk1s1 bytes_free=1239i,bytes_used=219833i 1667181610000000000 disk,host=host1,device=disk1s2 bytes_free=2234i,bytes_used=319838i 1667181600000000000 disk,host=host1,device=disk1s2 bytes_free=2239i,bytes_used=319833i 1667181610000000000 disk,host=host1,device=disk1s5 bytes_free=3234i,bytes_used=419838i 1667181600000000000 disk,host=host1,device=disk1s5 bytes_free=3239i,bytes_used=419833i 1667181610000000000 m2,tag0=val00 f64=0.98 1667181600000000000 m2,tag0=val02 f64=1.98 1667181600000000000 m2,tag0=val01 f64=2.98 1667181600000000000 m2,tag0=val05 f64=3.98 1667181600000000000 m2,tag0=val03 f64=4.98 1667181600000000000 m2,tag0=val09 f64=5.98 1667181600000000000 m2,tag0=val10 f64=6.98 1667181600000000000 m2,tag0=val08 f64=7.98 1667181600000000000 m2,tag0=val07 f64=8.98 1667181600000000000 m2,tag0=val04 f64=9.98 1667181600000000000 m3,tag0=a,tag1=b,tag2=c,tag3=d u64=1u 1667181600000000000 m4,tag.one=foo field.one=1 1667181600000000000 time_test,tt_tag=before_default_cutoff,tt_tag_before_default_cutoff=a tt_field="before_default_cutoff",tt_field_before_default_cutoff=1 631151999999999999 time_test,tt_tag=at_default_cutoff,tt_tag_at_default_cutoff=a tt_field="at_default_cutoff",tt_field_at_default_cutoff=1 631152000000000000 time_test,tt_tag=late,tt_tag_late=1 tt_field="late",tt_field_late=1 1667181600000000000 select_test,tag0=a,tag1=a,st_tag=aa,st_tag_aa=x st_field="aa",st_field_aa=1 1667181600000000000 select_test,tag0=a,tag1=b,st_tag=ab,st_tag_ab=x st_field="ab",st_field_ab=1 1667181600000000000 select_test,tag0=b,tag1=a,st_tag=ba,st_tag_ba=x st_field="ba",st_field_ba=1 1667181600000000000 select_test,tag0=b,tag1=b,st_tag=bb,st_tag_bb=x st_field="bb",st_field_bb=1 1667181600000000000 "# .to_string(), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, ], ), ( "DuplicateDifferentDomains", (0..2) .flat_map(|_| { [ Step::RecordNumParquetFiles, Step::WriteLineProtocol( r#" m,tag=A f=1 0 m,tag=A f=2 86400000000000 "#.into(), ), Step::Persist, Step::WaitForPersisted { expected_increase: 2, }, Step::RecordNumParquetFiles, Step::WriteLineProtocol( r#" m,tag=A f=3 1 "#.into(), ), Step::Persist, Step::WaitForPersisted { expected_increase: 1, }, ] .into_iter() }) .collect::>(), ), ]) }); /// Holds parameters for retention period. Tests based on this need to /// be run within the hour of being created. /// /// ```text (cut off) (now) /// time: -----------------------|--------------------------|> /// /// partially_inside: |-----------------| /// /// fully_inside: +1h |-----------| /// /// fully_outside: |---------| /// ``` /// /// Note this setup is only good for 1 hour after it was created. struct RetentionSetup { /// the retention period, relative to now() that the three data /// chunks fall inside/outside retention_period_ns: i64, /// lineprotocol data partially inside retention lp_partially_inside_1: String, /// lineprotocol data partially inside retention lp_partially_inside_2: String, /// lineprotocol data fully inside (included in query) lp_fully_inside: String, /// lineprotocol data fully outside (excluded from query) lp_fully_outside: String, } impl RetentionSetup { fn new() -> Self { let retention_period_1_hour_ns = 3600 * 1_000_000_000; // Data is relative to this particular time stamp // // Use a cutoff date that is NOT at the start of the partition so that `lp_partially_inside` only spans a single // partition, not two. This is important because otherwise this will result in two chunks / files, not one. // However a partial inside/outside chunk is important for the query tests so that we can proof that it is not // sufficient to prune the chunks solely on statistics but that there needs to be an actual row-wise filter. let cutoff = Time::from_rfc3339("2022-01-01T10:00:00+00:00") .unwrap() .timestamp_nanos(); // Timestamp 1 hour later than the cutoff, so the data will be retained for 1 hour let inside_retention = cutoff + retention_period_1_hour_ns; let outside_retention = cutoff - 10; // before retention let lp_partially_inside_1 = format!( "cpu,host=a load=1 {inside_retention}\n\ cpu,host=aa load=11 {outside_retention}" ); let lp_fully_inside = format!( "cpu,host=b load=2 {inside_retention}\n\ cpu,host=bb load=21 {inside_retention}" ); let lp_fully_outside = format!( "cpu,host=c load=3 {outside_retention}\n\ cpu,host=cc load=31 {outside_retention}" ); let lp_partially_inside_2 = format!( "cpu,host=d load=4 {inside_retention}\n\ cpu,host=dd load=41 {outside_retention}" ); // Set retention period to be at the cutoff date. Note that // since real world time advances, after 1 hour of real world // time the data that is inside the retention interval will // move outside (and thus not appear in queries). // // Thus this setup is only valid for 1 hour. let retention_period_ns = SystemProvider::new().now().timestamp_nanos() - cutoff; Self { // translate the retention period to be relative to now retention_period_ns, lp_partially_inside_1, lp_partially_inside_2, lp_fully_inside, lp_fully_outside, } } }