Fix health check when disconnect to etcd (#12292)

Signed-off-by: dragondriver <jiquan.long@zilliz.com>

cmd/roles/healthz_handler.go

@@ -84,5 +84,15 @@ func (handler *componentsHealthzHandler) ServeHTTP(w http.ResponseWriter, r *htt
 		return
 	}
 
+	if states.State == nil {
+		unhealthyHandler(w, r, "failed to get state")
+		return
+	}
+
+	if states.State.StateCode != internalpb.StateCode_Healthy {
+		unhealthyHandler(w, r, fmt.Sprintf("state: %s", states.State.StateCode.String()))
+		return
+	}
+
 	healthyHandler(w, r)
 }

internal/datanode/data_node.go

@@ -644,6 +644,10 @@ func (node *DataNode) Stop() error {
 	}
 
 	node.session.Revoke(time.Second)
+
+	// https://github.com/milvus-io/milvus/issues/12282
+	node.UpdateStateCode(internalpb.StateCode_Abnormal)
+
 	return nil
 }
 

internal/indexcoord/index_coord.go

@@ -282,6 +282,10 @@ func (i *IndexCoord) Stop() error {
 		cb()
 	}
 	i.session.Revoke(time.Second)
+
+	// https://github.com/milvus-io/milvus/issues/12282
+	i.UpdateStateCode(internalpb.StateCode_Abnormal)
+
 	return nil
 }
 

internal/indexnode/indexnode.go

@@ -223,6 +223,10 @@ func (i *IndexNode) Stop() error {
 		cb()
 	}
 	i.session.Revoke(time.Second)
+
+	// https://github.com/milvus-io/milvus/issues/12282
+	i.UpdateStateCode(internalpb.StateCode_Abnormal)
+
 	log.Debug("Index node stopped.")
 	return nil
 }

internal/proxy/proxy.go

@@ -369,6 +369,9 @@ func (node *Proxy) Stop() error {
 
 	node.session.Revoke(time.Second)
 
+	// https://github.com/milvus-io/milvus/issues/12282
+	node.UpdateStateCode(internalpb.StateCode_Abnormal)
+
 	return nil
 }