diff --git a/iox_data_generator/schemas/storage_cardinality_example.toml b/iox_data_generator/schemas/storage_cardinality_example.toml index 24b606b471..ef242d09ea 100644 --- a/iox_data_generator/schemas/storage_cardinality_example.toml +++ b/iox_data_generator/schemas/storage_cardinality_example.toml @@ -3,54 +3,54 @@ base_seed = "this is a demo" [[values]] name = "role" -value = "storage" +template = "storage" cardinality = 1 [[values]] name = "url" -value = "http://127.0.0.1:6060/metrics/usage" +template = "http://127.0.0.1:6060/metrics/usage" cardinality = 1 [[values]] name = "org_id" # Fill in the value with the cardinality counter and 15 random alphanumeric characters -value = "{{id}}_{{random 15}}" -cardinality = 1000 +template = "{{id}}_{{random 15}}" +cardinality = 1 has_one = ["env"] [[values]] name = "env" -value = "whatever-environment-{{id}}" +template = "whatever-environment-{{id}}" cardinality = 10 [[values]] name = "bucket_id" belongs_to = "org_id" # each bucket will have a unique value so it can be used here to guarantee uniqueness even across orgs -value = "{{id}}_{{random 15}}" +template = "{{id}}_{{random 15}}" # For each org, 3 buckets will be generated cardinality = 3 [[values]] name = "node_id" -value = "{{id}}" +template = "{{id}}" cardinality = 100 [[values]] name = "host" -value = "storage-{{node_id.value}}" +template = "storage-{{node_id.value}}" belongs_to = "node_id" cardinality = 1 [[values]] name = "hostname" -value = "{{node_id.value}}" +template = "{{node_id.value}}" belongs_to = "node_id" cardinality = 1 [[values]] name = "partition_id" -value = "{{id}}" +template = "{{id}}" cardinality = 10 # makes a tagset so every bucket exists on every node with every partition. @@ -75,16 +75,7 @@ name = "metric-scraper" [[agents.measurements]] name = "storage_usage_bucket_cardinality" -# each unique tag set will get an entry in this measurement -# tag_set = "bucket_set" -# -#[[agents.measurements.tags]] -#name = "role" -#value = "storage" -# -#[[agents.measurements.tags]] -#name = "url" -#value = "http://127.0.0.1:6060/metrics/usage" +# TODO: new syntax to make use of tag sets here [[agents.measurements.fields]] name = "gauge" diff --git a/iox_data_generator/src/specification.rs b/iox_data_generator/src/specification.rs index 818a9f5378..9637f03a4f 100644 --- a/iox_data_generator/src/specification.rs +++ b/iox_data_generator/src/specification.rs @@ -94,7 +94,7 @@ pub struct ValuesSpec { /// The name of the collection of values pub name: String, /// The handlebars template to create each value in the collection - pub value: String, + pub template: String, /// How many of these values should be generated. If belongs_to is /// specified, each parent will have this many of this value. So /// the total number of these values generated would be parent.len() * self.cardinality @@ -125,9 +125,10 @@ impl ValuesSpec { pub struct TagSetsSpec { /// The name of the tag set spec pub name: String, - /// An array of the values to loop through. To reference parent belongs_to or has_one - /// values, the parent should come first and then the has_one or child next. See the - /// doc for `ForEachValueTag` and its `value` for more detail. + /// An array of the `ValuesSpec` to loop through. To reference parent belongs_to or has_one + /// values, the parent should come first and then the has_one or child next. Each successive + /// entry in this array is a nested loop. Multiple has_one and a belongs_to on a parent can + /// be traversed. pub for_each: Vec, } diff --git a/iox_data_generator/src/tag_set.rs b/iox_data_generator/src/tag_set.rs index 18acae61c3..c4471ad469 100644 --- a/iox_data_generator/src/tag_set.rs +++ b/iox_data_generator/src/tag_set.rs @@ -58,19 +58,23 @@ pub struct GeneratedValueCollection { #[derive(Debug)] pub struct GeneratedValue { - value: Arc, id: usize, tag_pair: Arc, } #[derive(Debug, Default)] pub struct GeneratedTagSets { + // These map the name of a collection of values to its values. All values will have + // an entry in this map. For has_one and child_values, they will have duplicates there + // as well to make generating the tag sets possible. values: BTreeMap>>, // each parent-child will have its children stored in this map. The children map // the id of the parent to the collection of its children values child_values: BTreeMap>>>, // each parent-has_one will have its has_ones stored in this map has_one_values: BTreeMap, + // this maps the name of the tag set specified in the spec to the collection of tag + // sets that were pre-generated. tag_sets: BTreeMap>, } @@ -81,12 +85,6 @@ pub struct ParentToHasOnes { id_to_has_ones: BTreeMap, Arc>>, } -#[derive(Debug)] -pub struct TagSetKey { - value: String, - tag_key: Arc, -} - impl GeneratedTagSets { #[allow(dead_code)] pub fn from_spec(spec: &DataSpec) -> Result { @@ -295,7 +293,7 @@ impl GeneratedTagSets { spec: &ValuesSpec, ) -> Result<()> { template - .register_template_string(&spec.name, &spec.value) + .register_template_string(&spec.name, &spec.template) .context(CantCompileTemplate { template: &spec.name, })?; @@ -319,7 +317,6 @@ impl GeneratedTagSets { vals.push(Arc::new(GeneratedValue { id: i, - value: Arc::clone(&value), tag_pair: Arc::new(TagPair { key: Arc::clone(&tag_key), value, @@ -393,7 +390,7 @@ impl GeneratedTagSets { let data = json!({ belongs_to: { "id": parent.id, - "value": &parent.value, + "value": &parent.tag_pair.value, }, "id": child_value_id, }); @@ -408,7 +405,6 @@ impl GeneratedTagSets { let child_value = Arc::new(GeneratedValue { id: child_value_id, - value: Arc::clone(&value), tag_pair: Arc::new(TagPair { key: Arc::clone(&tag_key), value, @@ -488,7 +484,7 @@ base_seed = "foo" [[values]] name = "foo" -value = "{{id}}#foo" +template = "{{id}}#foo" cardinality = 3 [[tag_sets]] @@ -525,12 +521,12 @@ base_seed = "foo" [[values]] name = "foo" -value = "{{id}}#foo" +template = "{{id}}#foo" cardinality = 2 [[values]] name = "bar" -value = "{{id}}-{{foo.id}}-{{foo.value}}" +template = "{{id}}-{{foo.id}}-{{foo.value}}" cardinality = 2 belongs_to = "foo" @@ -572,30 +568,30 @@ base_seed = "foo" [[values]] name = "foo" -value = "{{id}}-foo" +template = "{{id}}-foo" cardinality = 3 has_one = ["bar"] [[values]] name = "bar" -value = "{{id}}-bar" +template = "{{id}}-bar" cardinality = 2 [[values]] name = "asdf" -value = "{{id}}-asdf" +template = "{{id}}-asdf" cardinality = 2 belongs_to = "foo" has_one = ["qwer"] [[values]] name = "jkl" -value = "{{id}}-jkl" +template = "{{id}}-jkl" cardinality = 2 [[values]] name = "qwer" -value = "{{id}}-qwer" +template = "{{id}}-qwer" cardinality = 6 [[tag_sets]]