Drop fail jobs on indexnode (#20425) (#20496)

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

internal/indexcoord/index_builder.go

@@ -345,6 +345,10 @@ func (ib *indexBuilder) process(buildID UniqueID) bool {
 			updateStateFunc(buildID, indexTaskDeleted)
 			return true
 		}
+		if !ib.dropIndexTask(buildID, meta.NodeID) {
+			log.Ctx(ib.ctx).RatedWarn(5, "drop index task fail, need retry")
+			return true
+		}
 		if err := ib.releaseLockAndResetTask(buildID, meta.NodeID); err != nil {
 			// release lock failed, no need to modify state, wait to retry
 			log.Ctx(ib.ctx).RatedWarn(10, "index builder try to release reference lock failed", zap.Error(err))

internal/indexcoord/index_builder_test.go

@@ -758,6 +758,26 @@ func TestIndexBuilder_Error(t *testing.T) {
 		ib.process(buildID)
 	})
 
+	t.Run("drop retry job fail", func(t *testing.T) {
+		ib.tasks[buildID] = indexTaskRetry
+		ib.meta = createMetaTable(&indexcoord.Catalog{
+			Txn: NewMockEtcdKV(),
+		})
+		err := ib.meta.UpdateVersion(buildID, nodeID)
+		assert.NoError(t, err)
+		ib.ic.nodeManager = &NodeManager{
+			ctx: context.Background(),
+			nodeClients: map[UniqueID]types.IndexNode{
+				nodeID: &indexnode.Mock{
+					CallDropJobs: func(ctx context.Context, in *indexpb.DropJobsRequest) (*commonpb.Status, error) {
+						return nil, errors.New("error")
+					},
+				},
+			},
+		}
+		ib.process(buildID)
+	})
+
 	t.Run("drop index job fail", func(t *testing.T) {
 		ib.tasks[buildID] = indexTaskDone
 		ib.meta = createMetaTable(&indexcoord.Catalog{