diff --git a/docs/examples/sample/gang-scheduling/README.md b/docs/examples/sample/gang-scheduling/README.md index e1a967e2a..bd2163999 100644 --- a/docs/examples/sample/gang-scheduling/README.md +++ b/docs/examples/sample/gang-scheduling/README.md @@ -75,5 +75,10 @@ This ensures that resources for all pods are reserved before the leader starts, To assign all groups of a LeaderWorkerSet to a specific Volcano queue, add the `scheduling.volcano.sh/queue-name` annotation to the LeaderWorkerSet metadata. + +### Volcano Scheduler Configurations + +To configure certain annotations in the PodGroup that are used by the Volcano scheduler to control specific scheduling behaviors, the PodGroup created by the LeaderWorkerSet will inherit the annotations starting with `volcano.sh/` from the LeaderWorkerSet's metadata.annotations. + For a complete example, please refer to the [lws-sample-volcano.yaml](./lws-sample-volcano.yaml) file. diff --git a/docs/examples/sample/gang-scheduling/lws-sample-volcano.yaml b/docs/examples/sample/gang-scheduling/lws-sample-volcano.yaml index 506954535..430ebda3c 100644 --- a/docs/examples/sample/gang-scheduling/lws-sample-volcano.yaml +++ b/docs/examples/sample/gang-scheduling/lws-sample-volcano.yaml @@ -3,6 +3,7 @@ kind: LeaderWorkerSet metadata: # annotations: # scheduling.volcano.sh/queue-name: lws-queue # Uncomment the annotation to set the queue name if needed + # volcano.sh/sla-waiting-time: 5m # PodGroups can inherit annotations with the prefix volcano.sh/. name: lws-sample-volcano spec: replicas: 2 diff --git a/pkg/schedulerprovider/volcano_provider.go b/pkg/schedulerprovider/volcano_provider.go index 8c1ccd318..deb8315ba 100644 --- a/pkg/schedulerprovider/volcano_provider.go +++ b/pkg/schedulerprovider/volcano_provider.go @@ -18,6 +18,7 @@ package schedulerprovider import ( "context" + "strings" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,7 +32,8 @@ import ( ) const ( - Volcano ProviderType = "volcano" + Volcano ProviderType = "volcano" + VolcanoAnnotationPrefix string = "volcano.sh/" ) type VolcanoProvider struct { @@ -63,6 +65,7 @@ func (v *VolcanoProvider) CreatePodGroupIfNotExists(ctx context.Context, lws *le leaderworkerset.GroupIndexLabelKey: leaderPod.Labels[leaderworkerset.GroupIndexLabelKey], leaderworkerset.RevisionKey: leaderPod.Labels[leaderworkerset.RevisionKey], }, + Annotations: inheritVolcanoAnnotations(lws), }, Spec: volcanov1beta1.PodGroupSpec{ // Default startupPolicy is LeaderCreated, Leader and Workers Pods are scheduled together, so MinAvailable is set to size @@ -104,3 +107,13 @@ func (v *VolcanoProvider) InjectPodGroupMetadata(pod *corev1.Pod) error { return nil } + +func inheritVolcanoAnnotations(lws *leaderworkerset.LeaderWorkerSet) map[string]string { + res := map[string]string{} + for k, v := range lws.Annotations { + if strings.HasPrefix(k, VolcanoAnnotationPrefix) { + res[k] = v + } + } + return res +} diff --git a/pkg/schedulerprovider/volcano_provider_test.go b/pkg/schedulerprovider/volcano_provider_test.go index c9042946f..06fc1fee1 100644 --- a/pkg/schedulerprovider/volcano_provider_test.go +++ b/pkg/schedulerprovider/volcano_provider_test.go @@ -50,6 +50,7 @@ func TestVolcanoProvider_CreatePodGroupIfNotExists(t *testing.T) { testLeaderPod1 := createTestLeaderPod("test-lws-0", "default", "test-lws", "0", "abc123") testLeaderPod2 := createTestLeaderPod("test-lws-1", "default", "test-lws", "1", "def456") testLeaderPod3 := createTestLeaderPod("test-lws-0", "default", "test-lws", "0", "xyz789") + testLeaderPod4 := createTestLeaderPod("test-lws-2", "default-1", "test-lws-1", "2", "jkl012") tests := []struct { name string @@ -155,6 +156,52 @@ func TestVolcanoProvider_CreatePodGroupIfNotExists(t *testing.T) { Spec: volcanov1beta1.PodGroupSpec{MinMember: 3}, }, }, + { + name: "create podgroup inherit volcano annotations", + lws: &leaderworkerset.LeaderWorkerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-lws-1", + Namespace: "default-1", + Annotations: map[string]string{ + "volcano.sh/sla-waiting-time": "5m", + }, + }, + Spec: leaderworkerset.LeaderWorkerSetSpec{ + LeaderWorkerTemplate: leaderworkerset.LeaderWorkerTemplate{ + Size: ptr.To[int32](3), + WorkerTemplate: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "worker", Image: "nginx", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}}, + }}, + }, + }, + }, + }, + }, + leaderPod: testLeaderPod4, + expectError: false, + expectedPG: &volcanov1beta1.PodGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-lws-1-2-jkl012", + Namespace: "default-1", + Labels: map[string]string{ + leaderworkerset.GroupIndexLabelKey: "2", + leaderworkerset.SetNameLabelKey: "test-lws-1", + leaderworkerset.RevisionKey: "jkl012", + }, + Annotations: map[string]string{ + "volcano.sh/sla-waiting-time": "5m", + }, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(testLeaderPod4, corev1.SchemeGroupVersion.WithKind("Pod"))}, + }, + Spec: volcanov1beta1.PodGroupSpec{ + MinMember: 3, + MinResources: &corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")}, + }, + }, + }, { name: "generic error on getting podgroup", lws: &leaderworkerset.LeaderWorkerSet{