chore: pr cleanup on data generator

iox_data_generator/schemas/storage_cardinality_example.toml

@@ -3,54 +3,54 @@ base_seed = "this is a demo"
 
 [[values]]
 name = "role"
-value = "storage"
+template = "storage"
 cardinality = 1
 
 [[values]]
 name = "url"
-value = "http://127.0.0.1:6060/metrics/usage"
+template = "http://127.0.0.1:6060/metrics/usage"
 cardinality = 1
 
 [[values]]
 name = "org_id"
 # Fill in the value with the cardinality counter and 15 random alphanumeric characters
-value = "{{id}}_{{random 15}}"
-cardinality = 1000
+template = "{{id}}_{{random 15}}"
+cardinality = 1
 has_one = ["env"]
 
 [[values]]
 name = "env"
-value = "whatever-environment-{{id}}"
+template = "whatever-environment-{{id}}"
 cardinality = 10
 
 [[values]]
 name = "bucket_id"
 belongs_to = "org_id"
 # each bucket will have a unique value so it can be used here to guarantee uniqueness even across orgs
-value = "{{id}}_{{random 15}}"
+template = "{{id}}_{{random 15}}"
 # For each org, 3 buckets will be generated
 cardinality = 3
 
 [[values]]
 name = "node_id"
-value = "{{id}}"
+template = "{{id}}"
 cardinality = 100
 
 [[values]]
 name = "host"
-value = "storage-{{node_id.value}}"
+template = "storage-{{node_id.value}}"
 belongs_to = "node_id"
 cardinality = 1
 
 [[values]]
 name = "hostname"
-value = "{{node_id.value}}"
+template = "{{node_id.value}}"
 belongs_to = "node_id"
 cardinality = 1
 
 [[values]]
 name = "partition_id"
-value = "{{id}}"
+template = "{{id}}"
 cardinality = 10
 
 # makes a tagset so every bucket exists on every node with every partition.
@@ -75,16 +75,7 @@ name = "metric-scraper"
 
 [[agents.measurements]]
 name = "storage_usage_bucket_cardinality"
-# each unique tag set will get an entry in this measurement
-# tag_set = "bucket_set"
-#
-#[[agents.measurements.tags]]
-#name = "role"
-#value = "storage"
-#
-#[[agents.measurements.tags]]
-#name = "url"
-#value = "http://127.0.0.1:6060/metrics/usage"
+# TODO: new syntax to make use of tag sets here
 
 [[agents.measurements.fields]]
 name = "gauge"

iox_data_generator/src/specification.rs

@@ -94,7 +94,7 @@ pub struct ValuesSpec {
     /// The name of the collection of values
     pub name: String,
     /// The handlebars template to create each value in the collection
-    pub value: String,
+    pub template: String,
     /// How many of these values should be generated. If belongs_to is
     /// specified, each parent will have this many of this value. So
     /// the total number of these values generated would be parent.len() * self.cardinality
@@ -125,9 +125,10 @@ impl ValuesSpec {
 pub struct TagSetsSpec {
     /// The name of the tag set spec
     pub name: String,
-    /// An array of the values to loop through. To reference parent belongs_to or has_one
-    /// values, the parent should come first and then the has_one or child next. See the
-    /// doc for `ForEachValueTag` and its `value` for more detail.
+    /// An array of the `ValuesSpec` to loop through. To reference parent belongs_to or has_one
+    /// values, the parent should come first and then the has_one or child next. Each successive
+    /// entry in this array is a nested loop. Multiple has_one and a belongs_to on a parent can
+    /// be traversed.
     pub for_each: Vec<String>,
 }
 

iox_data_generator/src/tag_set.rs

@@ -58,19 +58,23 @@ pub struct GeneratedValueCollection {
 
 #[derive(Debug)]
 pub struct GeneratedValue {
-    value: Arc<String>,
     id: usize,
     tag_pair: Arc<TagPair>,
 }
 
 #[derive(Debug, Default)]
 pub struct GeneratedTagSets {
+    // These map the name of a collection of values to its values. All values will have
+    // an entry in this map. For has_one and child_values, they will have duplicates there
+    // as well to make generating the tag sets possible.
     values: BTreeMap<String, Vec<Arc<GeneratedValue>>>,
     // each parent-child will have its children stored in this map. The children map
     // the id of the parent to the collection of its children values
     child_values: BTreeMap<String, BTreeMap<usize, Vec<Arc<GeneratedValue>>>>,
     // each parent-has_one will have its has_ones stored in this map
     has_one_values: BTreeMap<String, ParentToHasOnes>,
+    // this maps the name of the tag set specified in the spec to the collection of tag
+    // sets that were pre-generated.
     tag_sets: BTreeMap<String, Vec<TagSet>>,
 }
 
@@ -81,12 +85,6 @@ pub struct ParentToHasOnes {
     id_to_has_ones: BTreeMap<usize, BTreeMap<Arc<String>, Arc<GeneratedValue>>>,
 }
 
-#[derive(Debug)]
-pub struct TagSetKey {
-    value: String,
-    tag_key: Arc<String>,
-}
-
 impl GeneratedTagSets {
     #[allow(dead_code)]
     pub fn from_spec(spec: &DataSpec) -> Result<Self> {
@@ -295,7 +293,7 @@ impl GeneratedTagSets {
         spec: &ValuesSpec,
     ) -> Result<()> {
         template
-            .register_template_string(&spec.name, &spec.value)
+            .register_template_string(&spec.name, &spec.template)
             .context(CantCompileTemplate {
                 template: &spec.name,
             })?;
@@ -319,7 +317,6 @@ impl GeneratedTagSets {
 
                     vals.push(Arc::new(GeneratedValue {
                         id: i,
-                        value: Arc::clone(&value),
                         tag_pair: Arc::new(TagPair {
                             key: Arc::clone(&tag_key),
                             value,
@@ -393,7 +390,7 @@ impl GeneratedTagSets {
                 let data = json!({
                     belongs_to: {
                         "id": parent.id,
-                        "value": &parent.value,
+                        "value": &parent.tag_pair.value,
                     },
                     "id": child_value_id,
                 });
@@ -408,7 +405,6 @@ impl GeneratedTagSets {
 
                 let child_value = Arc::new(GeneratedValue {
                     id: child_value_id,
-                    value: Arc::clone(&value),
                     tag_pair: Arc::new(TagPair {
                         key: Arc::clone(&tag_key),
                         value,
@@ -488,7 +484,7 @@ base_seed = "foo"
 
 [[values]]
 name = "foo"
-value = "{{id}}#foo"
+template = "{{id}}#foo"
 cardinality = 3
 
 [[tag_sets]]
@@ -525,12 +521,12 @@ base_seed = "foo"
 
 [[values]]
 name = "foo"
-value = "{{id}}#foo"
+template = "{{id}}#foo"
 cardinality = 2
 
 [[values]]
 name = "bar"
-value = "{{id}}-{{foo.id}}-{{foo.value}}"
+template = "{{id}}-{{foo.id}}-{{foo.value}}"
 cardinality = 2
 belongs_to = "foo"
 
@@ -572,30 +568,30 @@ base_seed = "foo"
 
 [[values]]
 name = "foo"
-value = "{{id}}-foo"
+template = "{{id}}-foo"
 cardinality = 3
 has_one = ["bar"]
 
 [[values]]
 name = "bar"
-value = "{{id}}-bar"
+template = "{{id}}-bar"
 cardinality = 2
 
 [[values]]
 name = "asdf"
-value = "{{id}}-asdf"
+template = "{{id}}-asdf"
 cardinality = 2
 belongs_to = "foo"
 has_one = ["qwer"]
 
 [[values]]
 name = "jkl"
-value = "{{id}}-jkl"
+template = "{{id}}-jkl"
 cardinality = 2
 
 [[values]]
 name = "qwer"
-value = "{{id}}-qwer"
+template = "{{id}}-qwer"
 cardinality = 6
 
 [[tag_sets]]