diff --git a/deployments/monitor/grafana/milvus-dashboard.json b/deployments/monitor/grafana/milvus-dashboard.json index d15750edf0..3397707e2c 100644 --- a/deployments/monitor/grafana/milvus-dashboard.json +++ b/deployments/monitor/grafana/milvus-dashboard.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -18,22 +21,2592 @@ } ] }, + "description": "Recommend grafana with v8.5.20", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 34, - "iteration": 1667987821492, + "id": 110, "links": [], "liveNow": false, "panels": [ { - "collapsed": false, + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 123399, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Slow query count in 1 minute", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 123450, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "increase(milvus_proxy_slow_query_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Slow Query (in 1m)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "decimals": 0, + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 123397, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[1m])) by(function_name) ", + "interval": "", + "legendFormat": "{{function_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Successful Requests (in 1m)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:319", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:320", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 123411, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!~\"success|total\"}[1m])) by(function_name) ", + "interval": "", + "legendFormat": "{{function_name}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Failed Requests (in 1m)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:319", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:320", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 123405, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "legendFormat": "max-{{pod}}-{{node-id}}-{{msg_type}}", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Mutation Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of search request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 9 + }, + "hiddenSeries": false, + "id": 123404, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le,pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le,pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"search\"}[2m])))", + "hide": false, + "legendFormat": "max-{{pod}}-{{node-id}}", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Search Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of query request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 9 + }, + "hiddenSeries": false, + "id": 123410, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", query_type=\"query\"}[2m])))", + "hide": false, + "legendFormat": "max-{{pod}}-{{node_id}}", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Query Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 123402, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(milvus_rootcoord_quota_states{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (quota_states)", + "interval": "", + "legendFormat": "{{quota_states}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "RootCoord Quota State", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:376", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:377", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate limit request per second by status", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 17 + }, + "hiddenSeries": false, + "id": 123447, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(milvus_proxy_rate_limit_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[1m])) by (msg_type)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{msg_type}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "RateLimit Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:376", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:377", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "the channel cp lag, unit seconds", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 17 + }, + "hiddenSeries": false, + "id": 123434, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(timestamp(milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})-milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (channel_name)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{channel_name}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Channel Checkpoint Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The max time tick delay of flow graphs. unit ms", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 123407, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{role_name}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Time Tick Delay", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "ms", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "unit": "decmbytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 25 + }, + "hiddenSeries": false, + "id": 123412, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(milvus_querynode_disk_used_size{namespace=\"$namespace\",pod=~\"$instance-milvus.*\"} ) by(pod) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "decmbytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 25 + }, + "hiddenSeries": false, + "id": 123408, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(process_resident_memory_bytes{namespace=\"$namespace\",pod=~\"$instance-milvus.*\"} ) by(pod) / sum(container_spec_memory_limit_bytes{namespace=\"$namespace\",pod=~\"$instance-milvus.*\",image!=\"\",container!=\"\"} ) by(pod)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Usage Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 123409, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod=~\"$instance-milvus.*\",image!=\"\",container!=\"\"}[2m])) by (pod, namespace) / (sum(container_spec_cpu_quota{namespace=\"$namespace\",pod=~\"$instance-milvus.*\",image!=\"\",container!=\"\"}/100000) by (pod, namespace)) ", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:484", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:485", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of search request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 33 + }, + "hiddenSeries": false, + "id": 123468, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_restful_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[$__rate_interval])) by (phase) / sum(increase(milvus_proxy_restful_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[$__rate_interval])) by (phase)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{phase}}", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restful Search Latency Avg", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of search request over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 33 + }, + "hiddenSeries": false, + "id": 123469, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, phase) (rate(milvus_proxy_restful_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{phase}}", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restful Search Latency P99", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Service Quality", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 123455, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile of latency for meta operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 123453, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, meta_op_type) (rate(milvus_meta_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{meta_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Meta Operate Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile of latency for storage operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 2 + }, + "hiddenSeries": false, + "id": 123456, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, persistent_data_op_type) (rate(milvus_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operate Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile of latency for storage operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 2 + }, + "hiddenSeries": false, + "id": 123457, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, message_op_type) (rate(milvus_msgstream_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{message_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Message Operate Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Meta request rate", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 123458, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_meta_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (meta_op_type, status)", + "interval": "", + "legendFormat": "{{meta_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Meta Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage request rate", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "hiddenSeries": false, + "id": 123459, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (persistent_data_op_type, status)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Message request rate", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "hiddenSeries": false, + "id": 123460, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_msgstream_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (message_op_type, status)", + "interval": "", + "legendFormat": "{{message_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Message Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Meta operation throughput", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 123461, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_meta_kv_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (meta_op_type)", + "interval": "", + "legendFormat": "{{meta_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Meta Operation Throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "Bps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage operation throughput", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "hiddenSeries": false, + "id": 123462, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_storage_kv_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (persistent_data_op_type)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operation Throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "Bps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile of latency for storage operations", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "hiddenSeries": false, + "id": 123463, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, persistent_data_op_type) (rate(internal_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operate Latency[cpp]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage request rate in cpp", + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 123464, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (persistent_data_op_type, status)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}-{{status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Request Rate[cpp]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage operation throughput in cpp", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 26 + }, + "hiddenSeries": false, + "id": 123465, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(internal_storage_kv_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (persistent_data_op_type)", + "interval": "", + "legendFormat": "{{persistent_data_op_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Operation Throughput[cpp]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "Bps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Dependencies", + "type": "row" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, "id": 123248, "panels": [], "title": "Proxy", @@ -46,16 +2619,22 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of searching vectors.", + "fieldConfig": { + "defaults": { + "unit": "cps" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 1 + "y": 3 }, "hiddenSeries": false, "id": 123250, @@ -75,7 +2654,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -87,7 +2666,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", @@ -95,6 +2674,18 @@ "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m]))", + "hide": false, + "legendFormat": "total", + "range": true, + "refId": "B" } ], "thresholds": [], @@ -114,7 +2705,7 @@ "yaxes": [ { "$$hashKey": "object:3414", - "format": "short", + "format": "cps", "logBase": 1, "min": "0", "show": true @@ -137,16 +2728,22 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of inserting vectors.", + "fieldConfig": { + "defaults": { + "unit": "cps" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 1 + "y": 3 }, "hiddenSeries": false, "id": 123315, @@ -166,7 +2763,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -178,7 +2775,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", @@ -205,7 +2802,7 @@ "yaxes": [ { "$$hashKey": "object:3414", - "format": "short", + "format": "cps", "logBase": 1, "min": "0", "show": true @@ -228,16 +2825,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of search request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 1 + "y": 3 }, "hiddenSeries": false, "id": 123264, @@ -257,7 +2854,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -269,25 +2866,29 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", - "legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "range": true, "refId": "B" } ], @@ -308,7 +2909,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -331,16 +2932,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 7 + "y": 11 }, "hiddenSeries": false, "id": 123377, @@ -360,7 +2961,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -372,7 +2973,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_collection_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", @@ -384,7 +2985,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_collection_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_collection_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type)", @@ -411,7 +3012,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -434,16 +3035,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 7 + "y": 11 }, "hiddenSeries": false, "id": 123320, @@ -463,7 +3064,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -475,25 +3076,29 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.95, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, "refId": "B" } ], @@ -514,7 +3119,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -537,16 +3142,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 7 + "y": 11 }, "hiddenSeries": false, "id": 123378, @@ -566,7 +3171,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -578,7 +3183,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_collection_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))", @@ -590,7 +3195,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_collection_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type)", @@ -617,7 +3222,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -640,16 +3245,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of wait search result over the last 2 minutes.", + "description": "The 99th percentile and average search latency in step", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 13 + "y": 19 }, "hiddenSeries": false, "id": 123348, @@ -669,7 +3274,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -681,134 +3286,83 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "hide": false, "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-wait-result", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_wait_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_wait_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}-wait-result", + "range": true, "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Wait Search Result Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of reduce search result over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 13 - }, - "hiddenSeries": false, - "id": 123316, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, + "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", - "queryType": "randomWalk", - "refId": "A" + "hide": false, + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-reduce", + "range": true, + "refId": "C" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, + "editorMode": "code", "expr": "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}-reduce", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))\n", + "hide": false, + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}-decode", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", + "hide": false, + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}-decode", + "range": true, + "refId": "F" } ], "thresholds": [], "timeRegions": [], - "title": "Reduce Search Result Latency", + "title": "Search Latency In Step", "tooltip": { "shared": true, "sort": 0, @@ -823,7 +3377,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -846,19 +3400,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of decode search result over the last 2 minutes.", + "description": "The 99th percentile and average latency of send mutation over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 13 + "x": 8, + "y": 19 }, "hiddenSeries": false, - "id": 123317, + "id": 123321, "legend": { "avg": false, "current": false, @@ -875,7 +3429,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -887,32 +3441,35 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "hide": false, + "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "p99{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)", + "expr": "sum(increase(milvus_proxy_mutation_send_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_send_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Decode Search Result Latency", + "title": "Mutation Send Latency", "tooltip": { "shared": true, "sort": 0, @@ -927,7 +3484,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -950,16 +3507,436 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of mq mutation over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 19 + }, + "hiddenSeries": false, + "id": 123448, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod) (rate(milvus_msgstream_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", message_op_type=\"produce\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_msgstream_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", message_op_type=\"produce\"}[2m])) by (pod) / sum(increase(milvus_msgstream_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", message_op_type=\"produce\"}[2m])) by (pod)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "MQ Send Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 123327, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Apply Timestamp Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "ms", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile of latency for applying PK over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "hiddenSeries": false, + "id": 123326, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Apply PK Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of rpc in queue over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "hiddenSeries": false, + "id": 123470, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, function_name) (rate(milvus_proxy_req_in_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{function_name}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_proxy_req_in_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, function_name) / sum(increase(milvus_proxy_req_in_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, function_name)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{function_name}}", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "In Queue Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, "description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 19 + "y": 35 }, "hiddenSeries": false, "id": 123319, @@ -979,7 +3956,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -991,7 +3968,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -1004,7 +3981,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -1016,7 +3993,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -1066,314 +4043,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of send mutation over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 19 - }, - "hiddenSeries": false, - "id": 123321, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Mutation Send Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Average cache hits per minute of cache operations.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 19 - }, - "hiddenSeries": false, - "id": 123322, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(cache_name, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{cache_name}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cache Hit rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "short", - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of update cache over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 123323, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Cache Update Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3538", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3539", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 25 + "y": 35 }, "hiddenSeries": false, "id": 123325, @@ -1393,7 +4072,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1405,7 +4084,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -1419,7 +4098,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -1431,7 +4110,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -1443,7 +4122,7 @@ { "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, "exemplar": true, "expr": "", @@ -1493,19 +4172,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile of latency for applying PK over the last 2 minutes.", + "description": "The 99th percentile of latency for assigning segment id over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 25 + "y": 35 }, "hiddenSeries": false, - "id": 123326, + "id": 123449, "legend": { "avg": false, "current": false, @@ -1522,7 +4201,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1534,10 +4213,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_assign_segmentID_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}", "queryType": "randomWalk", @@ -1546,10 +4225,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "expr": "sum(increase(milvus_proxy_assign_segmentID_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_assign_segmentID_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}", @@ -1558,7 +4237,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Apply PK Latency", + "title": "Assign Segment ID Latency", "tooltip": { "shared": true, "sort": 0, @@ -1573,7 +4252,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -1596,19 +4275,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.", + "description": "The 99th percentile and average latency of request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 31 + "y": 43 }, "hiddenSeries": false, - "id": 123327, + "id": 123381, "legend": { "avg": false, "current": false, @@ -1625,7 +4304,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1637,31 +4316,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "legendFormat": "p99-{{function_name}}-{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "expr": "sum(increase(milvus_proxy_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "avg-{{function_name}}-{{pod}}-{{node_id}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Apply Timestamp Latency", + "title": "Request Latency", "tooltip": { "shared": true, "sort": 0, @@ -1676,7 +4355,7 @@ "yaxes": [ { "$$hashKey": "object:3538", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -1699,16 +4378,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of success requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 31 + "y": 43 }, "hiddenSeries": false, "id": 123329, @@ -1728,7 +4407,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1740,7 +4419,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[2m])/120) by(function_name, pod, node_id)", @@ -1790,16 +4469,301 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of update cache over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "hiddenSeries": false, + "id": 123323, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cache Update Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3538", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3539", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 51 + }, + "hiddenSeries": false, + "id": 123369, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(milvus_proxy_send_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[30s])) by(pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Send Byte Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3414", + "format": "binBps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3415", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of bytes received in proxy", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 51 + }, + "hiddenSeries": false, + "id": 123368, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(milvus_proxy_receive_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by(pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Received Byte Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:3414", + "format": "binBps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:3415", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, "description": "per-second increasing rate of faild requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 31 + "y": 51 }, "hiddenSeries": false, "id": 123379, @@ -1819,7 +4783,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1831,7 +4795,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by(function_name, pod, node_id)", @@ -1881,19 +4845,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of request over the last 2 minutes.", + "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 37 + "y": 59 }, "hiddenSeries": false, - "id": 123381, + "id": 123436, "legend": { "avg": false, "current": false, @@ -1910,7 +4874,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -1922,31 +4886,63 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "avg(milvus_proxy_limiter_rate{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, msg_type)", + "hide": true, "interval": "", - "legendFormat": "p99-{{function_name}}-{{pod}}-{{node_id}}", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}-avg", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(milvus_proxy_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name)", + "expr": "max(milvus_proxy_limiter_rate{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, msg_type)", + "hide": true, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}-max", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(milvus_proxy_limiter_rate{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, msg_type)", "hide": false, "interval": "", - "legendFormat": "avg-{{function_name}}-{{pod}}-{{node_id}}", - "refId": "B" + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}-min", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "D" } ], "thresholds": [], "timeRegions": [], - "title": "Request Latency", + "title": "Rate Limit", "tooltip": { "shared": true, "sort": 0, @@ -1960,14 +4956,14 @@ }, "yaxes": [ { - "$$hashKey": "object:3538", - "format": "short", + "$$hashKey": "object:536", + "decimals": 0, + "format": "bytes", "logBase": 1, - "min": "0", "show": true }, { - "$$hashKey": "object:3539", + "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true @@ -1978,194 +4974,119 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "Average cache hits per minute of cache operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] }, - "description": "per-second increasing rate of bytes received in proxy", - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 37 + "y": 59 }, - "hiddenSeries": false, - "id": 123368, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 123322, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_proxy_receive_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)", + "expr": "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(cache_name, pod, node_id)", "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-{{cache_name}}", "queryType": "randomWalk", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Received Byte Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 37 - }, - "hiddenSeries": false, - "id": 123369, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_proxy_send_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Send Byte Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "Cache Hit rate", + "type": "timeseries" }, { "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 43 + "y": 67 }, "id": 123155, "panels": [], @@ -2179,16 +5100,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Number of proxy nodes which has register with etcd", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 44 + "y": 68 }, "hiddenSeries": false, "id": 123141, @@ -2208,7 +5129,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2220,7 +5141,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -2250,7 +5171,6 @@ "$$hashKey": "object:4353", "format": "short", "logBase": 1, - "min": "0", "show": true }, { @@ -2271,16 +5191,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 44 + "y": 68 }, "hiddenSeries": false, "id": 123384, @@ -2300,7 +5220,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2312,7 +5232,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -2326,7 +5246,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -2338,7 +5258,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -2388,16 +5308,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 44 + "y": 68 }, "hiddenSeries": false, "id": 123338, @@ -2417,7 +5337,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2429,7 +5349,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -2441,7 +5361,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", @@ -2491,16 +5411,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 50 + "y": 76 }, "hiddenSeries": false, "id": 123337, @@ -2520,7 +5440,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2532,7 +5452,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -2544,7 +5464,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)", @@ -2571,7 +5491,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -2594,16 +5514,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "RootCoord stores pre-assigned timestamps in the metastore", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 50 + "y": 76 }, "hiddenSeries": false, "id": 123340, @@ -2623,7 +5543,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2635,14 +5555,16 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "expr": "sum(milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "interval": "", - "intervalFactor": 2, - "legendFormat": "tiestamp", + "intervalFactor": 1, + "legendFormat": "timestamp", "queryType": "randomWalk", + "range": true, "refId": "A" } ], @@ -2686,16 +5608,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of DDL request", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 50 + "y": 76 }, "hiddenSeries": false, "id": 123347, @@ -2715,7 +5637,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2727,7 +5649,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)", @@ -2777,16 +5699,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "RoootCoord current latest timestamp", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 56 + "y": 84 }, "hiddenSeries": false, "id": 123339, @@ -2806,7 +5728,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2818,14 +5740,16 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "expr": "sum(milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", "interval": "", - "intervalFactor": 2, - "legendFormat": "tiestamp", + "intervalFactor": 1, + "legendFormat": "timestamp", "queryType": "randomWalk", + "range": true, "refId": "A" } ], @@ -2869,16 +5793,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The number of DML channels", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 56 + "y": 84 }, "hiddenSeries": false, "id": 123345, @@ -2898,7 +5822,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -2910,7 +5834,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -2961,16 +5885,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of IDs assigned by RootCoord", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 56 + "y": 84 }, "hiddenSeries": false, "id": 123221, @@ -2990,7 +5914,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3002,7 +5926,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)", @@ -3052,16 +5976,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The number of partitions.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 62 + "y": 92 }, "hiddenSeries": false, "id": 123344, @@ -3081,7 +6005,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3093,7 +6017,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -3144,16 +6068,22 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The max time tick delay of flow graphs. unit ms", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 62 + "y": 92 }, "hiddenSeries": false, "id": 123383, @@ -3173,7 +6103,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3185,10 +6115,11 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", "interval": "", + "expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", + "interval": "", "intervalFactor": 2, "legendFormat": "{{role_name}}-{{node_id}}", "queryType": "randomWalk", @@ -3212,7 +6143,7 @@ "yaxes": [ { "$$hashKey": "object:4353", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -3235,16 +6166,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The number of collections.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 62 + "y": 92 }, "hiddenSeries": false, "id": 123342, @@ -3264,7 +6195,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3276,7 +6207,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -3327,16 +6258,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The number of credential", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 68 + "y": 100 }, "hiddenSeries": false, "id": 123343, @@ -3356,7 +6287,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3368,7 +6299,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -3419,16 +6350,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The number of Msgstream objects.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 68 + "y": 100 }, "hiddenSeries": false, "id": 123346, @@ -3448,7 +6379,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -3460,7 +6391,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -3505,977 +6436,588 @@ } }, { - "collapsed": true, + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 74 + "y": 108 }, "id": 123246, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of loaded collections.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 3 - }, - "hiddenSeries": false, - "id": 123288, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Collection Loaded Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of loaded entities.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 3 - }, - "hiddenSeries": false, - "id": 123289, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Entity Loaded Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of load requests.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 3 - }, - "hiddenSeries": false, - "id": 123291, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", - "interval": "", - "legendFormat": "{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of relaase requests.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 9 - }, - "hiddenSeries": false, - "id": 123292, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_release_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", - "interval": "", - "legendFormat": "{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Release Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of load request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 9 - }, - "hiddenSeries": false, - "id": 123294, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-latency", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_load_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_load_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg-latency", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load Request Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of release request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 9 - }, - "hiddenSeries": false, - "id": 123313, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-latency", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_release_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_release_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg-latency", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Release Request Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of sub-load task.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 123295, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_child_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Sub-Load Task", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number pf parent loading task.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 15 - }, - "hiddenSeries": false, - "id": 123296, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_parent_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Parent Load Task", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of sub-load task request over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 15 - }, - "hiddenSeries": false, - "id": 123298, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-latency", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querycoord_child_task_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_child_task_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))", - "hide": false, - "interval": "", - "legendFormat": "avg-latency", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Sub-Load Task Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of Query nodes which has register with etcd.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 123297, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "num", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Query Node Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], + "panels": [], "title": "Query Coordinator", "type": "row" }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of loaded collections.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 109 + }, + "hiddenSeries": false, + "id": 123288, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "num", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Collection Loaded Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of loaded collections.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 109 + }, + "hiddenSeries": false, + "id": 123403, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(milvus_querycoord_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance,querycoord_task_type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{querycoord_task_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Task Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of load requests.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 109 + }, + "hiddenSeries": false, + "id": 123291, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", + "interval": "", + "legendFormat": "{{status}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Load Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of Query nodes which has register with etcd.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 117 + }, + "hiddenSeries": false, + "id": 123297, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "num", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Query Node Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Segment Num On Stopping Node", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 117 + }, + "hiddenSeries": false, + "id": 123439, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(milvus_querynode_stopping_balance_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Segment Num On Stopping Node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Channel Num On Stopping Node", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 117 + }, + "hiddenSeries": false, + "id": 123444, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(milvus_querynode_stopping_balance_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Channel Num On Stopping Node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 75 + "y": 125 }, "id": 123244, "panels": [], @@ -4489,16 +7031,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Total number of the loaded collections in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 76 + "y": 126 }, "hiddenSeries": false, "id": 123299, @@ -4518,7 +7060,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -4530,7 +7072,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -4582,16 +7124,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Total number of the loaded partitions in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 76 + "y": 126 }, "hiddenSeries": false, "id": 123303, @@ -4611,7 +7153,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -4623,7 +7165,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -4675,16 +7217,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Total number of the loaded segment in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 76 + "y": 126 }, "hiddenSeries": false, "id": 123305, @@ -4704,19 +7246,19 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, segment_state)", @@ -4768,16 +7310,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 82 + "y": 134 }, "hiddenSeries": false, "id": 123385, @@ -4797,7 +7339,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -4809,7 +7351,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", @@ -4823,7 +7365,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", @@ -4835,7 +7377,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)", @@ -4885,330 +7427,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "Average, maximum and minimum values of the timestamps for time tick behind.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 82 - }, - "hiddenSeries": false, - "id": 123386, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TimeTick Lag Behind Now (Consumed Delete)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of consuming message", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 82 - }, - "hiddenSeries": false, - "id": 123387, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-all", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Consumed Message Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "cps", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of the queryable entities in QueryNode.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 88 - }, - "hiddenSeries": false, - "id": 123365, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Queryable Entity Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Total number of dml virtual channels for QueryNode watch", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 88 + "y": 134 }, "hiddenSeries": false, "id": 123304, @@ -5228,7 +7456,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5240,7 +7468,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -5291,19 +7519,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of delta virtual channels for QueryNode watch", + "description": "per-second increasing rate of consuming message", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 88 + "y": 134 }, "hiddenSeries": false, - "id": 123306, + "id": 123387, "legend": { "avg": false, "current": false, @@ -5320,7 +7548,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5332,21 +7560,32 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-all", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Delta Virtual Channel", + "title": "Consumed Message Rate", "tooltip": { "shared": true, "sort": 0, @@ -5360,14 +7599,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "format": "short", + "$$hashKey": "object:3414", + "format": "cps", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -5384,19 +7623,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of consumers in the QueryNode.", + "description": "The 99th percentile and average latency of delegator process insert/delete over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 94 + "y": 142 }, "hiddenSeries": false, - "id": 123307, + "id": 123438, "legend": { "avg": false, "current": false, @@ -5413,7 +7652,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5425,21 +7664,141 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, msg_type) (rate(milvus_querynode_process_insert_or_delete_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(milvus_querynode_process_insert_or_delete_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, msg_type) / sum(increase(milvus_querynode_process_insert_or_delete_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, msg_type)", "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" + "legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Consumer Num", + "title": "Processing Insert/Delete Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of the queryable entities in QueryNode.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 142 + }, + "hiddenSeries": false, + "id": 123365, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, collection_id, segment_state)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-{{collection_id}}-{{segment_state}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", + "hide": false, + "legendFormat": "Total", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Queryable Entity Num", "tooltip": { "shared": true, "sort": 0, @@ -5454,6 +7813,7 @@ "yaxes": [ { "$$hashKey": "object:536", + "decimals": 0, "format": "short", "logBase": 1, "min": "0", @@ -5477,16 +7837,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of searching requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 94 + "x": 16, + "y": 142 }, "hiddenSeries": false, "id": 123350, @@ -5506,7 +7866,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5518,7 +7878,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)", @@ -5568,122 +7928,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.", + "description": "search latency by phase", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 94 - }, - "hiddenSeries": false, - "id": 123366, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Search Request Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 100 + "y": 150 }, "hiddenSeries": false, - "id": 123372, + "id": 123466, "legend": { "avg": false, "current": false, @@ -5700,7 +7957,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5712,31 +7969,81 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "p99-{{query_type}}-InQueue", "queryType": "randomWalk", + "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "editorMode": "code", + "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type) ", "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "legendFormat": "avg-{{query_type}}-InQueue", + "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "legendFormat": "p99-{{query_type}}-{{segment_state}}-Segment", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type, segment_state)", + "hide": false, + "legendFormat": "avg-{{query_type}}-{{segment_state}}-Segment", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "legendFormat": "p99-{{query_type}}-Reduce", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (query_type)) ", + "hide": false, + "legendFormat": "avg-{{query_type}}-Reduce", + "range": true, + "refId": "F" } ], "thresholds": [], "timeRegions": [], - "title": "Search in Queue Latency", + "title": "Search Latency By Phase", "tooltip": { "shared": true, "sort": 0, @@ -5750,14 +8057,14 @@ }, "yaxes": [ { - "$$hashKey": "object:161", - "format": "short", + "$$hashKey": "object:3414", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:162", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -5774,16 +8081,22 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of search segment over the last 2 minutes.", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 100 + "y": 150 }, "hiddenSeries": false, "id": 123374, @@ -5803,7 +8116,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5815,7 +8128,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": false, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -5827,7 +8140,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)", @@ -5854,7 +8167,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -5877,19 +8190,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", + "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 100 + "y": 150 }, "hiddenSeries": false, - "id": 123310, + "id": 123366, "legend": { "avg": false, "current": false, @@ -5906,7 +8219,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -5918,10 +8231,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", @@ -5930,10 +8243,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", @@ -5942,7 +8255,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Segcore Request Latency", + "title": "Search Request Latency", "tooltip": { "shared": true, "sort": 0, @@ -5957,7 +8270,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -5980,19 +8293,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.", + "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 106 + "y": 158 }, "hiddenSeries": false, - "id": 123367, + "id": 123372, "legend": { "avg": false, "current": false, @@ -6009,7 +8322,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6021,10 +8334,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", @@ -6033,10 +8346,10 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", @@ -6045,7 +8358,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Search Reduce Latency", + "title": "Search in Queue Latency", "tooltip": { "shared": true, "sort": 0, @@ -6060,7 +8373,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -6083,16 +8396,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of load segment over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 106 + "y": 158 }, "hiddenSeries": false, "id": 123311, @@ -6112,7 +8425,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6124,7 +8437,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -6136,7 +8449,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -6163,7 +8476,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -6186,19 +8499,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "Total number of flowgraph", + "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 106 + "y": 158 }, "hiddenSeries": false, - "id": 123312, + "id": 123310, "legend": { "avg": false, "current": false, @@ -6215,7 +8528,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6227,20 +8540,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Flowgraph Num", + "title": "Segcore Request Latency", "tooltip": { "shared": true, "sort": 0, @@ -6254,14 +8578,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -6278,19 +8602,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The length of the task queue for unsolved read requests", + "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 112 + "y": 166 }, "hiddenSeries": false, - "id": 123351, + "id": 123367, "legend": { "avg": false, "current": false, @@ -6307,7 +8631,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6319,20 +8643,31 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Unsolved Read Task Length", + "title": "Search Reduce Latency", "tooltip": { "shared": true, "sort": 0, @@ -6346,14 +8681,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", - "format": "short", + "$$hashKey": "object:161", + "format": "ms", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true @@ -6370,16 +8705,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The length of the task queue of read requests to be executed", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 112 + "y": 166 }, "hiddenSeries": false, "id": 123356, @@ -6399,7 +8734,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6411,7 +8746,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -6462,16 +8797,612 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 166 + }, + "hiddenSeries": false, + "id": 123435, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_wait_tsafe_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_sq_wait_tsafe_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_wait_tsafe_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Wait tSafe Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The length of the task queue for unsolved read requests", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 174 + }, + "hiddenSeries": false, + "id": 123351, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Unsolved Read Task Length", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of queries for search requests", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 174 + }, + "hiddenSeries": false, + "id": 123361, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Search NQ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of flowgraph", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 174 + }, + "hiddenSeries": false, + "id": 123312, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Flowgraph Num", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of queries for the merged search requests", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 182 + }, + "hiddenSeries": false, + "id": 123360, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-milvus_querynode_search_group_nq", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Search Group NQ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of original tasks contained in the merged search task", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 182 + }, + "hiddenSeries": false, + "id": 123352, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Search Group Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, "description": "Number of read requests currently being executed in parallel", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 112 + "y": 182 }, "hiddenSeries": false, "id": 123357, @@ -6491,7 +9422,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6503,7 +9434,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -6554,19 +9485,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "cpu utilization under scheduler evaluation", + "description": "per-second increasing rate of evicted read requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 118 + "y": 190 }, "hiddenSeries": false, - "id": 123358, + "id": 123364, "legend": { "avg": false, "current": false, @@ -6583,7 +9514,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6595,12 +9526,11 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", + "expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", "interval": "", - "intervalFactor": 2, "legendFormat": "{{pod}}-{{node_id}}", "queryType": "randomWalk", "refId": "A" @@ -6608,7 +9538,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Estimate CPU Usage", + "title": "Evicted Read Requests Rate", "tooltip": { "shared": true, "sort": 0, @@ -6622,14 +9552,14 @@ }, "yaxes": [ { - "$$hashKey": "object:536", + "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { - "$$hashKey": "object:537", + "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true @@ -6646,325 +9576,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "The number of original tasks contained in the merged search task", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 118 - }, - "hiddenSeries": false, - "id": 123352, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Search Group Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of queries for search requests", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 118 - }, - "hiddenSeries": false, - "id": 123361, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Search NQ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of queries for the merged search requests", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 124 - }, - "hiddenSeries": false, - "id": 123360, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-milvus_querynode_search_group_nq", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Search Group NQ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Top_K for search requests", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 124 + "y": 190 }, "hiddenSeries": false, "id": 123359, @@ -6984,7 +9605,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -6996,7 +9617,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -7008,7 +9629,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -7058,16 +9679,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Top_K for the merged search requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 124 + "y": 190 }, "hiddenSeries": false, "id": 123362, @@ -7087,7 +9708,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7099,7 +9720,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -7111,7 +9732,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -7154,405 +9775,1065 @@ "align": false } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of evicted read requests.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 130 - }, - "hiddenSeries": false, - "id": 123364, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Evicted Read Requests Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:3414", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:3415", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "id": 123397, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 130 - }, - "type": "graph", - "title": "Knowhere Search Top_K", - "datasource": { - "uid": "$datasource", - "type": "prometheus" - }, - "thresholds": [], - "pluginVersion": "8.5.20", - "description": "Top_K for knowhere search requests", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "aliasColors": {}, - "dashLength": 10, - "fill": 1, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "pointradius": 2, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(knowhere_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(knowhere_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(knowhere_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "timeRegions": [], - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "xaxis": { - "mode": "time", - "show": true, - "values": [], - "name": null, - "buckets": null - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - }, - "bars": false, - "dashes": false, - "fillGradient": 0, - "hiddenSeries": false, - "percentage": false, - "points": false, - "stack": false, - "steppedLine": false, - "timeFrom": null, - "timeShift": null - }, - { - "id": 123395, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 130 - }, - "type": "graph", - "title": "Knowhere Search Count", - "datasource": { - "uid": "$datasource", - "type": "prometheus" - }, - "thresholds": [], - "pluginVersion": "8.5.20", - "description": "Total number of knowhere search cnt.", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "aliasColors": {}, - "dashLength": 10, - "fill": 1, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "pointradius": 2, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(knowhere_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeRegions": [], - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "xaxis": { - "mode": "time", - "show": true, - "values": [], - "name": null, - "buckets": null - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - }, - "bars": false, - "dashes": false, - "fillGradient": 0, - "hiddenSeries": false, - "percentage": false, - "points": false, - "stack": false, - "steppedLine": false, - "timeFrom": null, - "timeShift": null - }, - { - "id": 123396, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 136 - }, - "type": "graph", - "title": "Knowhere Range Search Count", - "datasource": { - "uid": "$datasource", - "type": "prometheus" - }, - "thresholds": [], - "pluginVersion": "8.5.20", - "description": "Total number of knowhere range search cnt.", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "aliasColors": {}, - "dashLength": 10, - "fill": 1, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "pointradius": 2, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(knowhere_range_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeRegions": [], - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "xaxis": { - "mode": "time", - "show": true, - "values": [], - "name": null, - "buckets": null - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - }, - "bars": false, - "dashes": false, - "fillGradient": 0, - "hiddenSeries": false, - "percentage": false, - "points": false, - "stack": false, - "steppedLine": false, - "timeFrom": null, - "timeShift": null - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 136 + "y": 198 + }, + "id": 123419, + "panels": [], + "title": "Knowhere", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 199 + }, + "id": 123420, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(search_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(search_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(search_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "search latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 199 + }, + "id": 123421, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(range_search_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(range_search_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(range_search_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "range search latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 199 + }, + "id": 123428, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(build_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(build_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(knowhere_build_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "build index latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 207 + }, + "id": 123425, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(search_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, module)", + "legendFormat": "{{module}}-{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "search rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 207 + }, + "id": 123423, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(range_search_latencycount{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, module)", + "legendFormat": "{{module}}-{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "range search rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 207 + }, + "id": 123417, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(build_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, module)", + "legendFormat": "{{module}}-{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "build index rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 215 + }, + "id": 123443, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, module, pod) (rate(load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "instant": false, + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(load_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (module, pod) / sum(increase(load_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (module, pod)", + "hide": false, + "instant": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "Load Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 215 + }, + "id": 123427, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(ann_iterator_init_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(ann_iterator_init_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(ann_iterator_init_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "ann iterator init latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 215 + }, + "id": 123429, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod) (rate(knowhere_index_version_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(knowhere_index_version_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod) / sum(increase(knowhere_index_version_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod)", + "hide": false, + "legendFormat": "avg-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "knowhere create version index latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 223 + }, + "id": 123433, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le,pod, module) (rate(search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "legendFormat": "p99-{{module}}-{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module) / sum(increase(search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, module)", + "hide": false, + "legendFormat": "avg-{{module}}-{{pod}}", + "range": true, + "refId": "B" + } + ], + "title": "search topk ", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 231 }, "id": 123172, "panels": [], @@ -7566,16 +10847,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Number of data nodes which has register with etcd.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 137 + "y": 232 }, "hiddenSeries": false, "id": 123207, @@ -7595,7 +10876,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7607,7 +10888,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -7658,16 +10939,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" + "uid": "${datasource}" }, "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 137 + "y": 232 }, "hiddenSeries": false, "id": 123390, @@ -7687,7 +10968,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7699,7 +10980,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -7713,7 +10994,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -7725,7 +11006,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -7775,16 +11056,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The number of rows of valid data accumulated in DataCoord that flushed.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 137 + "y": 232 }, "hiddenSeries": false, "id": 123269, @@ -7804,7 +11085,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7816,7 +11097,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -7867,16 +11148,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of data flushed.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 143 + "y": 240 }, "hiddenSeries": false, "id": 123371, @@ -7896,7 +11177,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7908,7 +11189,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)", @@ -7958,16 +11239,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Number of segments with different states in the Meta of DataCoord", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 143 + "y": 240 }, "hiddenSeries": false, "id": 123267, @@ -7987,7 +11268,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -7999,7 +11280,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)", @@ -8050,16 +11331,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "binlog size of all collections/segments, unit byte", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 143 + "y": 240 }, "hiddenSeries": false, "id": 123382, @@ -8079,7 +11360,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8091,7 +11372,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", @@ -8135,13 +11416,294 @@ "align": false } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 248 + }, + "hiddenSeries": false, + "id": 123437, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "topK(10, timestamp(milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) - milvus_datacoord_channel_checkpoint_unix_seconds{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, channel_name)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-avg", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "channel cp lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:536", + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:537", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of building index requests recived by IndexCoord.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 248 + }, + "hiddenSeries": false, + "id": 123225, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(milvus_datacoord_index_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (status)", + "interval": "", + "legendFormat": "{{status}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Index Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:372", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:373", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of building index requests recived by IndexCoord.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 248 + }, + "hiddenSeries": false, + "id": 123451, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(milvus_datacoord_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[1m])) by (index_task_status)", + "interval": "", + "legendFormat": "{{index_task_status}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Index Task Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:372", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:373", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, { "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 149 + "y": 256 }, "id": 123242, "panels": [], @@ -8155,16 +11717,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "Total number of flowgraph", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 150 + "y": 257 }, "hiddenSeries": false, "id": 123272, @@ -8184,7 +11746,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8196,7 +11758,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", @@ -8247,342 +11809,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of prodecer created on the DataNode", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 150 - }, - "hiddenSeries": false, - "id": 123277, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Producer Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 150 - }, - "hiddenSeries": false, - "id": 123393, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TimeTick Lag Behind Now (Consumed All)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "P1809F7CD0C75ACF3" - }, - "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now, which send to DataCoord", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 156 - }, - "hiddenSeries": false, - "id": 123388, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "avg(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}-avg", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "max(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-max", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "min(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-min", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Produced TimeTick Lag Behind Now For DC", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "decimals": 0, - "format": "ms", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of consuming message", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 156 + "y": 257 }, "hiddenSeries": false, "id": 123389, @@ -8602,7 +11838,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8614,7 +11850,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)", @@ -8664,19 +11900,19 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "forward delete and timetick message to delta channel latency", + "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 156 + "y": 257 }, "hiddenSeries": false, - "id": 123394, + "id": 123393, "legend": { "avg": false, "current": false, @@ -8693,7 +11929,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -8705,31 +11941,45 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_forward_delete_msg_time_taken_ms_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "expr": "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", "hide": false, "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", + "intervalFactor": 2, + "legendFormat": "{{pod}}-{{node_id}}-avg", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, - "expr": "sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "expr": "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)", "hide": false, "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", + "legendFormat": "{{pod}}-{{node_id}}-min", "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "Forward Delete&Timetick Message latency", + "title": "TimeTick Lag Behind Now (Consumed All)", "tooltip": { "shared": true, "sort": 0, @@ -8767,576 +12017,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of consumers created on the DataNode", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 162 - }, - "hiddenSeries": false, - "id": 123276, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Consumer Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 162 - }, - "hiddenSeries": false, - "id": 123282, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Encode Buffer Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of messages consumed for insert and delete operation.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 162 - }, - "hiddenSeries": false, - "id": 123274, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Msg Rows Consumed Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "cps", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total number of segment that has been not flushed.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 168 - }, - "hiddenSeries": false, - "id": 123280, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Unflushed Segment Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:536", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:537", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of auto flush operate.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 168 - }, - "hiddenSeries": false, - "id": 123285, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Autoflush Operate Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of each message that has been flushed.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 168 - }, - "hiddenSeries": false, - "id": 123275, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", - "interval": "", - "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Flush Data Size Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of flush operete.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 174 + "y": 265 }, "hiddenSeries": false, "id": 123284, @@ -9356,7 +12046,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -9368,7 +12058,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", @@ -9418,16 +12108,496 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 265 + }, + "hiddenSeries": false, + "id": 123282, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Encode Buffer Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of messages consumed for insert and delete operation.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 265 + }, + "hiddenSeries": false, + "id": 123274, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Msg Rows Consumed Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "cps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The 99th percentile and average latency of compaction over the last 2 minutes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 273 + }, + "hiddenSeries": false, + "id": 123314, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", + "hide": false, + "interval": "", + "legendFormat": "p99-{{pod}}-{{node_id}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", + "hide": false, + "interval": "", + "legendFormat": "avg-{{pod}}-{{node_id}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Compaction Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:161", + "format": "ms", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:162", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of auto flush operate.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 273 + }, + "hiddenSeries": false, + "id": 123285, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{status}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Autoflush Operate Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "per-second increasing rate of each message that has been flushed.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 273 + }, + "hiddenSeries": false, + "id": 123275, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)", + "interval": "", + "legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Flush Data Size Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:101", + "format": "binBps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:102", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, "description": "per-second increasing rate of flush requests.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 174 + "x": 0, + "y": 281 }, "hiddenSeries": false, "id": 123286, @@ -9447,7 +12617,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -9459,7 +12629,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", @@ -9509,16 +12679,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 174 + "x": 8, + "y": 281 }, "hiddenSeries": false, "id": 123283, @@ -9538,7 +12708,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -9550,7 +12720,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -9562,7 +12732,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -9589,7 +12759,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -9606,406 +12776,16 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "collapsed": true, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "description": "The 99th percentile and average latency of compaction over the last 2 minutes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 180 - }, - "hiddenSeries": false, - "id": 123314, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "hide": false, - "interval": "", - "legendFormat": "p99-{{pod}}-{{node_id}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", - "hide": false, - "interval": "", - "legendFormat": "avg-{{pod}}-{{node_id}}", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Compaction Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:161", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:162", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 186 - }, - "id": 123223, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "per-second increasing rate of building index requests recived by IndexCoord.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 123225, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(increase(milvus_indexcoord_index_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)", - "interval": "", - "legendFormat": "{{status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:372", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:373", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of indexing tasks with different states in IndexCoord's meta", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 7 - }, - "hiddenSeries": false, - "id": 123227, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_indexcoord_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (index_task_status)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{index_task_status}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index Task Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:401", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:402", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of index nodes which has register with etcd", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 7 - }, - "hiddenSeries": false, - "id": 123229, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(milvus_indexcoord_index_node_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "total", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Index Node Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:479", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:480", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "title": "Index Coordinator", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 187 + "y": 289 }, "id": 123231, "panels": [ @@ -10016,16 +12796,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "per-second increasing rate of index tasks recevied by IndexNode", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 8 + "y": 306 }, "hiddenSeries": false, "id": 123233, @@ -10045,7 +12825,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -10057,7 +12837,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)", @@ -10107,16 +12887,22 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes.", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 8 + "y": 306 }, "hiddenSeries": false, "id": 123235, @@ -10136,7 +12922,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -10148,7 +12934,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -10160,7 +12946,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_load_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_load_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -10187,7 +12973,7 @@ "yaxes": [ { "$$hashKey": "object:161", - "format": "short", + "format": "ms", "logBase": 1, "show": true }, @@ -10209,16 +12995,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of decod the FieldData over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 8 + "y": 306 }, "hiddenSeries": false, "id": 123238, @@ -10238,7 +13024,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -10250,7 +13036,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -10262,7 +13048,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_decode_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_decode_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -10312,16 +13098,22 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of build index over the last 2 minutes.", + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 0, - "y": 14 + "y": 314 }, "hiddenSeries": false, "id": 123237, @@ -10341,7 +13133,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -10353,7 +13145,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -10365,7 +13157,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_build_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_build_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -10392,7 +13184,7 @@ "yaxes": [ { "$$hashKey": "object:246", - "format": "short", + "format": "ms", "logBase": 1, "min": "0", "show": true @@ -10415,16 +13207,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile and average latency of encode index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 8, - "y": 14 + "y": 314 }, "hiddenSeries": false, "id": 123239, @@ -10444,7 +13236,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -10456,7 +13248,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -10468,7 +13260,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_encode_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_encode_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -10518,16 +13310,16 @@ "dashes": false, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "description": "The 99th percentile of latency for save index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 8, "x": 16, - "y": 14 + "y": 314 }, "hiddenSeries": false, "id": 123240, @@ -10547,7 +13339,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.3.3", + "pluginVersion": "10.2.0", "pointradius": 2, "points": false, "renderer": "flot", @@ -10559,7 +13351,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", @@ -10571,7 +13363,7 @@ { "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_save_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_save_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)", @@ -10618,1841 +13410,66 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 188 + "y": 290 }, "id": 123157, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Total cpu usage of all milvus components.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 183 - }, - "hiddenSeries": false, - "id": 123202, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m]))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "cpu usage", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage Total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:8411", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:8412", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 183 - }, - "hiddenSeries": false, - "id": 123204, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", - "interval": "", - "intervalFactor": 2, - "legendFormat": "memory", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1340", - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:1341", - "format": "bytes", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of goroutines that currently exist.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 183 - }, - "hiddenSeries": false, - "id": 123206, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", - "interval": "", - "intervalFactor": 2, - "legendFormat": "goroutines", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Goroutines Total", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1281", - "format": "short", - "label": "", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:1282", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "process cpu usage", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 189 - }, - "hiddenSeries": false, - "id": 123135, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Process memory of milvus pod.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 189 - }, - "hiddenSeries": false, - "id": 123133, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": false, - "expr": "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of goroutines that currently exist.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 189 - }, - "hiddenSeries": false, - "id": 123125, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of bytes allocated and still in use.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 195 - }, - "hiddenSeries": false, - "id": 123137, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Allocated Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "GC Max duration seconds.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 195 - }, - "hiddenSeries": false, - "id": 123131, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", - "interval": "", - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC Max duration seconds", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of OS threads created.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 195 - }, - "hiddenSeries": false, - "id": 123127, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "OS Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Next GC Bytes.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 201 - }, - "hiddenSeries": false, - "id": 123129, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Next GC Bytes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of process opened fds.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 201 - }, - "hiddenSeries": false, - "id": 123211, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Process Opened Fds", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "The fraction of this program's available CPU time used by the GC since the program started.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 201 - }, - "hiddenSeries": false, - "id": 123212, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "GC CPU time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Rate of memory frees.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 207 - }, - "hiddenSeries": false, - "id": 123213, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Free Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Rate of go memory mallocs.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 207 - }, - "hiddenSeries": false, - "id": 123215, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Memory mallocs rate.", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of allocated heap objects.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 207 - }, - "hiddenSeries": false, - "id": 123216, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Allocated Heap Objects", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Rate of bytes allocated, even if freed.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 213 - }, - "hiddenSeries": false, - "id": 123217, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Allocated Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:8539", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:8540", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of heap bytes released to OS.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 213 - }, - "hiddenSeries": false, - "id": 123218, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Heap released", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "description": "Number of heap bytes waiting to be used.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 213 - }, - "hiddenSeries": false, - "id": 123219, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Heap idle", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of heap bytes that be used.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 219 - }, - "hiddenSeries": false, - "id": 123375, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "go_memstats_heap_inuse_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Go Heap in Use", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of container threads created.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 219 - }, - "hiddenSeries": false, - "id": 123376, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.3.3", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "exemplar": true, - "expr": "container_threads{image=\"\", namespace=\"$namespace\"}", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Container Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], + "panels": [], "title": "Runtime", "type": "row" }, { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 123402, - "panels": [], - "title": "storage", - "type": "row" - }, - { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, + "description": "Total cpu usage of all milvus components.", + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 5 + "h": 8, + "w": 8, + "x": 0, + "y": 291 }, - "id": 123406, + "hiddenSeries": false, + "id": 123202, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { @@ -12460,1369 +13477,1652 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "legendFormat": "90% {{persistent_data_op_type}}", + "exemplar": true, + "expr": "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "cpu usage", + "queryType": "randomWalk", "range": true, "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:8411", + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, + { + "$$hashKey": "object:8412", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 291 + }, + "hiddenSeries": false, + "id": 123204, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "memory", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1340", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1341", + "format": "bytes", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of goroutines that currently exist.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 291 + }, + "hiddenSeries": false, + "id": 123206, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "goroutines", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Goroutines Total", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1281", + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1282", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "process cpu usage", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 299 + }, + "hiddenSeries": false, + "id": 123135, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.75, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", + "exemplar": true, + "expr": "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[1m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:544", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:545", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Process memory of milvus pod.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 299 + }, + "hiddenSeries": false, + "id": 123133, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", "hide": false, - "legendFormat": "75% {{persistent_data_op_type}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "hide": false, - "legendFormat": "50% {{persistent_data_op_type}}", - "range": true, - "refId": "C" - } - ], - "title": "[go]kv size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 15 - }, - "id": 123446, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "rate(milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"get\"}[2m])", - "legendFormat": "{{le}}", - "range": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]get", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "thresholds": [], + "timeRegions": [], + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 25 - }, - "id": 123447, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ + "yaxes": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "rate(milvus_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"put\"}[2m])", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" + "$$hashKey": "object:598", + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:599", + "format": "bytes", + "logBase": 1, + "show": true } ], - "title": "[go]put", - "type": "timeseries" + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "list::success", - "remove::total" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, + "description": "Number of goroutines that currently exist.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, + "w": 8, + "x": 16, + "y": 299 + }, + "hiddenSeries": false, + "id": 123125, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of bytes allocated and still in use.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, "x": 0, - "y": 35 + "y": 307 }, - "id": 123404, + "hiddenSeries": false, + "id": 123137, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, + "exemplar": true, + "expr": "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]total/success request", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Go Allocated Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "description": "GC Max duration seconds.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 35 + "w": 8, + "x": 8, + "y": 307 }, - "id": 123444, + "hiddenSeries": false, + "id": 123131, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, pod, persistent_data_op_type) (rate(milvus_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "legendFormat": "{{persistent_data_op_type}}", - "range": true, + "exemplar": true, + "expr": "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "interval": "", + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]pct99", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "GC Max duration seconds", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, + "description": "Number of OS threads created.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, + "w": 8, + "x": 16, + "y": 307 + }, + "hiddenSeries": false, + "id": 123127, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "OS Threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Next GC Bytes.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, "x": 0, - "y": 43 + "y": 315 }, - "id": 123405, + "hiddenSeries": false, + "id": 123129, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(milvus_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, + "exemplar": true, + "expr": "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]failed request", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Next GC Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "description": "Number of process opened fds.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 43 + "w": 8, + "x": 8, + "y": 315 }, - "id": 123407, + "hiddenSeries": false, + "id": 123211, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(milvus_storage_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type) / sum(increase(milvus_storage_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}", - "range": true, + "exemplar": true, + "expr": "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[go]avg", - "type": "timeseries" - },{ - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "thresholds": [], + "timeRegions": [], + "title": "Process Opened Fds", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] }, - "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 5 - }, - "id": 123506, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ + "yaxes": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "legendFormat": "90% {{persistent_data_op_type}}", - "range": true, - "refId": "A" + "format": "short", + "logBase": 1, + "min": "0", + "show": true }, { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "hide": false, - "legendFormat": "75% {{persistent_data_op_type}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"})", - "hide": false, - "legendFormat": "50% {{persistent_data_op_type}}", - "range": true, - "refId": "C" + "format": "short", + "logBase": 1, + "show": true } ], - "title": "[cpp]kv size", - "type": "timeseries" + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 15 - }, - "id": 123546, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "rate(internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"get\"}[2m])", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "[cpp]get", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 14, - "x": 5, - "y": 25 - }, - "id": 123547, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "rate(internal_storage_kv_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", persistent_data_op_type=\"put\"}[2m])", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "[cpp]put", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "list::success", - "remove::total" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, + "description": "The fraction of this program's available CPU time used by the GC since the program started.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, + "w": 8, + "x": 16, + "y": 315 + }, + "hiddenSeries": false, + "id": 123212, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "GC CPU time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Rate of memory frees.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, "x": 0, - "y": 35 + "y": 323 }, - "id": 123504, + "hiddenSeries": false, + "id": 123213, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status!=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, + "exemplar": true, + "expr": "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[cpp]total/success request", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Memory Free Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Rate of go memory mallocs.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 323 + }, + "hiddenSeries": false, + "id": 123215, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Go Memory mallocs rate.", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of allocated heap objects.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 323 + }, + "hiddenSeries": false, + "id": 123216, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Allocated Heap Objects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "description": "Rate of bytes allocated, even if freed.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 123544, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, pod, persistent_data_op_type) (rate(internal_storage_request_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))", - "legendFormat": "{{persistent_data_op_type}}", - "range": true, - "refId": "A" - } - ], - "title": "[cpp]pct99", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 43 + "y": 331 }, - "id": 123505, + "hiddenSeries": false, + "id": 123217, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "sum(increase(internal_storage_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by (status, pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}::{{status}}", - "range": true, + "exemplar": true, + "expr": "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "[cpp]failed request", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Memory Allocated Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:8539", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:8540", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of heap bytes released to OS.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 331 + }, + "hiddenSeries": false, + "id": 123218, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Go Heap released", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Number of heap bytes waiting to be used.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 331 + }, + "hiddenSeries": false, + "id": 123219, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Go Heap idle", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, + "description": "Number of heap bytes that be used.", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 43 + "w": 8, + "x": 0, + "y": 339 }, - "id": 123507, + "hiddenSeries": false, + "id": 123375, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "alertThreshold": true }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "go_memstats_heap_inuse_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Go Heap in Use", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of container threads created.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 339 + }, + "hiddenSeries": false, + "id": 123376, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { @@ -13830,30 +15130,59 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(internal_storage_request_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type) / sum(increase(internal_storage_request_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, persistent_data_op_type)", - "legendFormat": "{{persistent_data_op_type}}", + "exemplar": true, + "expr": "container_threads{image=\"\", namespace=\"$namespace\", pod=~\"$instance-.*\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "queryType": "randomWalk", "range": true, "refId": "A" } ], - "title": "[cpp]avg", - "type": "timeseries" + "thresholds": [], + "timeRegions": [], + "title": "Container Threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:420", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:421", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "refresh": false, - "schemaVersion": 34, - "style": "dark", + "refresh": "", + "schemaVersion": 38, "tags": [ - "milvus2.0" + "milvus2.4", + "milvus2.3" ], "templating": { "list": [ { - "current": { - "selected": false, - "text": "prometheus", - "value": "prometheus" - }, "hide": 0, "includeAll": false, "multi": false, @@ -13867,42 +15196,20 @@ "type": "datasource" }, { - "current": { - "selected": true, - "text": "milvus", - "value": "milvus" - }, - "hide": 2, - "name": "app_name", - "options": [ - { - "selected": true, - "text": "milvus", - "value": "milvus" - } - ], - "query": "milvus", - "skipUrlSync": false, - "type": "textbox" - }, - { - "current": { - "selected": true, - "text": "chaos-testing", - "value": "chaos-testing" - }, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info, namespace)", + "definition": "label_values(go_info,namespace)", "hide": 0, "includeAll": false, "multi": false, "name": "namespace", "options": [], "query": { - "query": "label_values(kube_pod_info, namespace)", - "refId": "prometheus-namespace-Variable-Query" + "qryType": 1, + "query": "label_values(go_info,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", @@ -13914,23 +15221,20 @@ "useTags": false }, { - "current": { - "selected": false, - "text": "bulk-insert-test", - "value": "bulk-insert-test" - }, "datasource": { - "uid": "$datasource" + "type": "prometheus", + "uid": "${datasource}" }, - "definition": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)", + "definition": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"},app_kubernetes_io_instance)", "hide": 0, "includeAll": false, "multi": false, "name": "instance", "options": [], "query": { - "query": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"},app_kubernetes_io_instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", @@ -13950,17 +15254,18 @@ }, "datasource": { "type": "prometheus", - "uid": "$datasource" + "uid": "${datasource}" }, - "definition": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)", + "definition": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\"},collection_name)", "hide": 0, "includeAll": true, "multi": false, "name": "collection", "options": [], "query": { - "query": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\"},collection_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", @@ -13970,60 +15275,22 @@ }, { "current": { - "selected": false, - "text": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps", - "value": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps" + "selected": true, + "text": "milvus", + "value": "milvus" }, - "datasource": { - "uid": "$datasource" - }, - "definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)", "hide": 2, - "includeAll": false, - "multi": true, - "name": "pod", - "options": [], - "query": { - "query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)", - "refId": "prometheus-pod-Variable-Query" - }, - "refresh": 2, - "regex": "", + "name": "app_name", + "options": [ + { + "selected": true, + "text": "milvus", + "value": "milvus" + } + ], + "query": "milvus", "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": { - "uid": "$datasource" - }, - "definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)", - "hide": 2, - "includeAll": false, - "multi": true, - "name": "component", - "options": [], - "query": { - "query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "textbox" } ] }, @@ -14060,7 +15327,7 @@ }, "timezone": "browser", "title": "Milvus2.0", - "uid": "uLf5cJ3Gz", - "version": 5, + "uid": "uLf5cJ3Ga", + "version": 3, "weekStart": "" }