fix(task): Use existing scheduler metrics (#15765)

task/backend/scheduler/scheduler_metrics.go

@@ -1,6 +1,8 @@
 package scheduler
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -103,6 +105,18 @@ func (em *SchedulerMetrics) release(taskID ID) {
 	em.releaseCalls.Inc()
 }
 
+func (em *SchedulerMetrics) reportScheduleDelay(d time.Duration) {
+	em.scheduleDelay.Observe(d.Seconds())
+}
+
+func (em *SchedulerMetrics) reportExecution(err error, d time.Duration) {
+	em.totalExecuteCalls.Inc()
+	em.executeDelta.Observe(d.Seconds())
+	if err != nil {
+		em.totalExecuteFailure.Inc()
+	}
+}
+
 func newExecutingTasks(ts *TreeScheduler) *executingTasks {
 	return &executingTasks{
 		desc: prometheus.NewDesc(

task/backend/scheduler/treescheduler.go

@@ -314,7 +314,14 @@ func (s *TreeScheduler) work(ctx context.Context, ch chan Item) {
 					err = &ErrUnrecoverable{errors.New("executor panicked")}
 				}
 			}()
-			return s.executor.Execute(ctx, it.id, t)
+			// report the difference between when the item was supposed to be scheduled and now
+			s.sm.reportScheduleDelay(time.Since(it.Next()))
+			preExec := time.Now()
+			// execute
+			err = s.executor.Execute(ctx, it.id, t)
+			// report how long execution took
+			s.sm.reportExecution(err, time.Since(preExec))
+			return err
 		}()
 		if err != nil {
 			s.onErr(ctx, it.id, it.Next(), err)