feat: record the number of times milvus turns into force-deny-writing states (#34960)

/kind improvement
fix: #34961

---------

Signed-off-by: longjiquan <jiquan.long@zilliz.com>

internal/rootcoord/quota_center.go

@@ -1569,6 +1569,7 @@ func (q *QuotaCenter) recordMetrics() {
 						return false
 					}
 					metrics.RootCoordQuotaStates.WithLabelValues(errorCode.String(), name).Set(1.0)
+					metrics.RootCoordForceDenyWritingCounter.Inc()
 					return false
 				}
 				return true

pkg/metrics/rootcoord_metrics.go

@@ -170,6 +170,15 @@ var (
 			"name",
 		})
 
+	// RootCoordForceDenyWritingCounter records the number of times that milvus turns into force-deny-writing states.
+	RootCoordForceDenyWritingCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.RootCoordRole,
+			Name:      "force_deny_writing_counter",
+			Help:      "The number of times milvus turns into force-deny-writing states",
+		})
+
 	// RootCoordRateLimitRatio reflects the ratio of rate limit.
 	RootCoordRateLimitRatio = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -241,6 +250,7 @@ func RegisterRootCoord(registry *prometheus.Registry) {
 	registry.MustRegister(RootCoordNumOfRoles)
 	registry.MustRegister(RootCoordTtDelay)
 	registry.MustRegister(RootCoordQuotaStates)
+	registry.MustRegister(RootCoordForceDenyWritingCounter)
 	registry.MustRegister(RootCoordRateLimitRatio)
 	registry.MustRegister(RootCoordDDLReqLatencyInQueue)