Add scheduler e2e latency metric (#767)

Author: liu-cong

pkg/epp/metrics/metrics.go

@@ -30,7 +30,6 @@ import (
 const (
 	InferenceModelComponent = "inference_model"
 	InferencePoolComponent  = "inference_pool"
-	EPPComponent            = "endpoint_picker"
 	InferenceExtension      = "inference_extension"
 )
 
@@ -184,10 +183,22 @@ var (
 		[]string{"name"},
 	)
 
-	// Scheduler Plugin Metrics
+	// Scheduler Metrics
+	SchedulerE2ELatency = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Subsystem: InferenceExtension,
+			Name:      "scheduler_e2e_duration_seconds",
+			Help:      "End-to-end scheduling latency distribution in seconds.",
+			Buckets: []float64{
+				0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1,
+			},
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{},
+	)
 	SchedulerPluginProcessingLatencies = compbasemetrics.NewHistogramVec(
 		&compbasemetrics.HistogramOpts{
-			Subsystem: EPPComponent,
+			Subsystem: InferenceExtension,
 			Name:      "scheduler_plugin_duration_seconds",
 			Help:      "Scheduler plugin processing latency distribution in seconds for each plugin type and plugin name.",
 			Buckets: []float64{
@@ -230,6 +241,7 @@ func Register() {
 		legacyregistry.MustRegister(inferencePoolReadyPods)
 
 		legacyregistry.MustRegister(SchedulerPluginProcessingLatencies)
+		legacyregistry.MustRegister(SchedulerE2ELatency)
 
 		legacyregistry.MustRegister(InferenceExtensionInfo)
 	})
@@ -335,6 +347,11 @@ func RecordSchedulerPluginProcessingLatency(pluginType, pluginName string, durat
 	SchedulerPluginProcessingLatencies.WithLabelValues(pluginType, pluginName).Observe(duration.Seconds())
 }
 
+// RecordSchedulerE2ELatency records the end-to-end scheduling latency.
+func RecordSchedulerE2ELatency(duration time.Duration) {
+	SchedulerE2ELatency.WithLabelValues().Observe(duration.Seconds())
+}
+
 func RecordInferenceExtensionInfo() {
 	if CommitSHA != "" {
 		InferenceExtensionInfo.WithLabelValues(CommitSHA).Set(1)

pkg/epp/metrics/metrics_test.go

@@ -614,7 +614,50 @@ func TestSchedulerPluginProcessingLatencies(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, wantPluginLatencies, "endpoint_picker_scheduler_plugin_processing_latencies"); err != nil {
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, wantPluginLatencies, "inference_extension_scheduler_plugin_duration_seconds"); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+func TestSchedulerE2ELatency(t *testing.T) {
+	scenarios := []struct {
+		name      string
+		durations []time.Duration
+	}{
+		{
+			name: "multiple scheduling latencies",
+			durations: []time.Duration{
+				200 * time.Microsecond,  // 0.00014s - should go in the 0.0002 bucket
+				800 * time.Microsecond,  // 0.0008s - should go in the 0.001 bucket
+				1500 * time.Microsecond, // 0.0015s - should go in the 0.002 bucket
+				3 * time.Millisecond,    // 0.003s - should go in the 0.005 bucket
+				8 * time.Millisecond,    // 0.008s - should go in the 0.01 bucket
+				15 * time.Millisecond,   // 0.015s - should go in the 0.02 bucket
+				30 * time.Millisecond,   // 0.03s - should go in the 0.05 bucket
+				75 * time.Millisecond,   // 0.075s - should go in the 0.1 bucket
+				150 * time.Millisecond,  // 0.15s - should go in the +Inf bucket
+			},
+		},
+	}
+	Register()
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			for _, duration := range scenario.durations {
+				RecordSchedulerE2ELatency(duration)
+			}
+
+			wantE2ELatency, err := os.Open("testdata/scheduler_e2e_duration_seconds_metric")
+			defer func() {
+				if err := wantE2ELatency.Close(); err != nil {
+					t.Error(err)
+				}
+			}()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, wantE2ELatency, "inference_extension_scheduler_e2e_duration_seconds"); err != nil {
 				t.Error(err)
 			}
 		})

pkg/epp/metrics/testdata/scheduler_e2e_duration_seconds_metric

@@ -0,0 +1,15 @@
+# HELP inference_extension_scheduler_e2e_duration_seconds [ALPHA] End-to-end scheduling latency distribution in seconds.
+# TYPE inference_extension_scheduler_e2e_duration_seconds histogram
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.0001"} 0
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.0002"} 1
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.0005"} 1
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.001"} 2
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.002"} 3
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.005"} 4
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.01"} 5
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.02"} 6
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.05"} 7
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="0.1"} 8
+inference_extension_scheduler_e2e_duration_seconds_bucket{le="+Inf"} 9
+inference_extension_scheduler_e2e_duration_seconds_sum{} 0.2835
+inference_extension_scheduler_e2e_duration_seconds_count{} 9