diff --git a/iox_data_generator/schemas/many_dbs.toml b/iox_data_generator/schemas/many_dbs.toml new file mode 100644 index 0000000000..8f160e7f0d --- /dev/null +++ b/iox_data_generator/schemas/many_dbs.toml @@ -0,0 +1,80 @@ +# This schema is meant to test out many databases writing data in like a bunch of free tier users. +# Start with a database_list of 10k to make things interesting. This will send on average of +# 208 requests/sec and 6.25 MB/sec across the 10k databases. The top 10 will have 60 requests/min +# and 1.8MB/min. +name = "many_dbs" + +[[values]] +name = "some_tag_10" +cardinality = 2 +template = "id_{{id}}_{{random 15}}" +has_one = ["extra_static"] + +[[values]] +name = "child_tag" +cardinality = 3 +belongs_to = "some_tag_10" +has_one = ["rotation"] +template = "id_{{id}}_{{random 10}}" + +[[values]] +name = "rotation" +cardinality = 4 +template = "id_{{id}}_{{guid}}" + +[[values]] +name = "extra_static" +cardinality = 1 +template = "whatever-constant-value" + +[[tag_sets]] +name = "first_set" +for_each = [ + "some_tag_10", + "some_tag_10.extra_static", + "some_tag_10.child_tag", + "child_tag.rotation", +] + +# each sampling from this agent generates 32,465 bytes of LP, first few lines look like: +# main_measurement_1,agent_id=1,child_tag=id_1_T6iJnnBTE3,extra_static=whatever-constant-value,measurement_tag=1,rotation=id_1_de4ddb8c-31a6-440f-a273-7132bdd43bd7,some_tag_10=id_1_rWtIkI26LTlfu0J intfield=71334i,floatfield=0.7934452557768101 1639151629935287000 +# main_measurement_1,agent_id=1,child_tag=id_2_VsiUF2xVuz,extra_static=whatever-constant-value,measurement_tag=1,rotation=id_2_890145b3-8157-4d6f-ac02-1fe37584190f,some_tag_10=id_1_rWtIkI26LTlfu0J intfield=64582i,floatfield=0.0957134480635704 1639151629935287000 +# main_measurement_1,agent_id=1,child_tag=id_3_XNL51f1NdT,extra_static=whatever-constant-value,measurement_tag=1,rotation=id_3_8bcf7547-06e9-4033-9ffb-e00ac4e6c5a9,some_tag_10=id_1_rWtIkI26LTlfu0J intfield=26179i,floatfield=0.09993902612184669 1639151629935287000 +# main_measurement_1,agent_id=1,child_tag=id_4_mqCyprcTDQ,extra_static=whatever-constant-value,measurement_tag=1,rotation=id_4_f465d43e-f1ab-4250-99ac-67af7c1d4c72,some_tag_10=id_2_X4eWjH9ImjTeta2 intfield=16511i,floatfield=0.033060266070114475 1639151629935287000 +[[agents]] +name = "first_agent" +tag_pairs = [ + {key = "agent_id", template = "{{agent.id}}"} +] + +[[agents.measurements]] +name = "main_measurement_{{measurement.id}}" +count = 20 +tag_set = "first_set" +tag_pairs = [ + {key = "measurement_tag", template = "{{measurement.id}}"} +] + +[[agents.measurements.fields]] +name = "intfield" +i64_range = [1, 100000] + +[[agents.measurements.fields]] +name = "floatfield" +f64_range = [0.0, 1.0] + +[[database_writers]] +database_ratio = 0.001 +agents = [{name = "first_agent", sampling_interval = "1s"}] + +[[database_writers]] +database_ratio = 0.01 +agents = [{name = "first_agent", sampling_interval = "10s"}] + +[[database_writers]] +database_ratio = 0.1 +agents = [{name = "first_agent", sampling_interval = "30s"}] + +[[database_writers]] +database_ratio = 1.0 +agents = [{name = "first_agent", sampling_interval = "60s"}] \ No newline at end of file diff --git a/iox_data_generator/schemas/many_measurements.toml b/iox_data_generator/schemas/many_measurements.toml new file mode 100644 index 0000000000..66bd27d51e --- /dev/null +++ b/iox_data_generator/schemas/many_measurements.toml @@ -0,0 +1,61 @@ +# This schema tests what load looks like with many measurements (2,000). If pointed at a single database +# with the configured 20 agents at 10s sampling, it will send an average of 2 requests/second (representing +# 16k rows) with 4.1MB/second of LP being written. Each agent writes 8k lines per request. +name = "many_measurements" + +[[values]] +name = "some_tag" +cardinality = 2 +template = "id_{{id}}_{{random 15}}" +has_one = ["extra_static"] + +[[values]] +name = "child_tag" +cardinality = 2 +belongs_to = "some_tag" +has_one = ["rotation"] +template = "id_{{id}}_{{random 10}}" + +[[values]] +name = "rotation" +cardinality = 3 +template = "id_{{id}}_{{guid}}" + +[[values]] +name = "extra_static" +cardinality = 1 +template = "whatever-constant-value" + +[[tag_sets]] +name = "first_set" +for_each = [ + "some_tag", + "some_tag.extra_static", + "some_tag.child_tag", + "child_tag.rotation", +] + +[[agents]] +name = "first_agent" +tag_pairs = [ + {key = "agent_id", template = "{{agent.id}}"} +] + +[[agents.measurements]] +name = "main_measurement_{{measurement.id}}" +count = 2000 +tag_set = "first_set" +tag_pairs = [ + {key = "measurement_id", template = "{{measurement.id}}"} +] + +[[agents.measurements.fields]] +name = "intfield" +i64_range = [1, 100000] + +[[agents.measurements.fields]] +name = "floatfield" +f64_range = [0.0, 1.0] + +[[database_writers]] +agents = [{name = "first_agent", sampling_interval = "10s", count = 20}]