refine stop order (#28016)

Signed-off-by: Wei Liu <wei.liu@zilliz.com>

cmd/roles/roles.go

@@ -351,8 +351,12 @@ func (mr *MilvusRoles) Run() {
 		rootCoord = mr.runRootCoord(ctx, local, &wg)
 	}
 
-	if mr.EnableProxy {
+	if mr.EnableDataCoord {
-		proxy = mr.runProxy(ctx, local, &wg)
+		dataCoord = mr.runDataCoord(ctx, local, &wg)
+	}
+
+	if mr.EnableIndexCoord {
+		indexCoord = mr.runIndexCoord(ctx, local, &wg)
 	}
 
 	if mr.EnableQueryCoord {
@@ -363,22 +367,17 @@ func (mr *MilvusRoles) Run() {
 		queryNode = mr.runQueryNode(ctx, local, &wg)
 	}
 
-	if mr.EnableDataCoord {
-		dataCoord = mr.runDataCoord(ctx, local, &wg)
-	}
-
 	if mr.EnableDataNode {
 		dataNode = mr.runDataNode(ctx, local, &wg)
 	}
-
-	if mr.EnableIndexCoord {
-		indexCoord = mr.runIndexCoord(ctx, local, &wg)
-	}
-
 	if mr.EnableIndexNode {
 		indexNode = mr.runIndexNode(ctx, local, &wg)
 	}
 
+	if mr.EnableProxy {
+		proxy = mr.runProxy(ctx, local, &wg)
+	}
+
 	wg.Wait()
 
 	mr.setupLogger()
@@ -390,40 +389,29 @@ func (mr *MilvusRoles) Run() {
 	<-mr.closed
 
 	// stop coordinators first
-	//	var component
+	coordinators := []component{rootCoord, dataCoord, indexCoord, queryCoord}
-	coordinators := []component{rootCoord, queryCoord, dataCoord, indexCoord}
 	for idx, coord := range coordinators {
 		log.Warn("stop processing")
 		if coord != nil {
-			log.Warn("stop coord", zap.Int("idx", idx), zap.Any("coord", coord))
+			log.Info("stop coord", zap.Int("idx", idx), zap.Any("coord", coord))
-			wg.Add(1)
+			coord.Stop()
-			go func(coord component) {
-				defer wg.Done()
-				coord.Stop()
-			}(coord)
 		}
 	}
-	wg.Wait()
 	log.Info("All coordinators have stopped")
 
 	// stop nodes
 	nodes := []component{queryNode, indexNode, dataNode}
-	for _, node := range nodes {
+	for idx, node := range nodes {
 		if node != nil {
-			wg.Add(1)
+			log.Info("stop node", zap.Int("idx", idx), zap.Any("node", node))
-			go func(node component) {
+			node.Stop()
-				defer wg.Done()
-				node.Stop()
-			}(node)
 		}
 	}
-	wg.Wait()
 	log.Info("All nodes have stopped")
 
-	// stop proxy
 	if proxy != nil {
 		proxy.Stop()
-		log.Info("proxy stopped")
+		log.Info("proxy stopped!")
 	}
 
 	// close reused etcd client

internal/distributed/rootcoord/service.go

@@ -311,6 +311,13 @@ func (s *Server) Stop() error {
 	if s.tikvCli != nil {
 		defer s.tikvCli.Close()
 	}
+
+	s.cancel()
+	if s.grpcServer != nil {
+		utils.GracefulStopGRPCServer(s.grpcServer)
+	}
+	s.wg.Wait()
+
 	if s.dataCoord != nil {
 		if err := s.dataCoord.Close(); err != nil {
 			log.Error("Failed to close dataCoord client", zap.Error(err))
@@ -326,12 +333,6 @@ func (s *Server) Stop() error {
 			log.Error("Failed to close close rootCoord", zap.Error(err))
 		}
 	}
-	log.Debug("Rootcoord begin to stop grpc server")
-	s.cancel()
-	if s.grpcServer != nil {
-		utils.GracefulStopGRPCServer(s.grpcServer)
-	}
-	s.wg.Wait()
 	return nil
 }
 

scripts/stop_graceful.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 function get_milvus_process() {
-    echo $(ps -e | grep milvus | grep -v grep | awk '{print $1}')
+    echo $(ps -e | grep milvus | grep -v grep |  grep run-with-subprocess | awk '{print $1}')
 }
 
 echo "Stopping milvus..."