diff --git a/pkg/webhooks/pod_webhook.go b/pkg/webhooks/pod_webhook.go
index 860ca83f2..ddce845c4 100644
--- a/pkg/webhooks/pod_webhook.go
+++ b/pkg/webhooks/pod_webhook.go
@@ -192,6 +192,9 @@ func SetExclusiveAffinities(pod *corev1.Pod, groupUniqueKey string, topologyKey
 		pod.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{}
 	}
 
+	// Get the LWS name from pod labels
+	lwsName := pod.Labels[leaderworkerset.SetNameLabelKey]
+
 	// Pod affinity ensures the pods of this set land on the same topology domain.
 	pod.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(pod.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution,
 		corev1.PodAffinityTerm{
@@ -204,10 +207,17 @@ func SetExclusiveAffinities(pod *corev1.Pod, groupUniqueKey string, topologyKey
 			}},
 			TopologyKey: topologyKey,
 		})
-	// Pod anti-affinity ensures exclusively this set lands on the topology, preventing multiple sets per topology domain.
+
+	// Pod anti-affinity ensures exclusively one replica/group of this set lands on the topology, preventing multiple replicas per topology domain.
+	// and only apply mutual exclusion within the same LWS instance
 	pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution,
 		corev1.PodAffinityTerm{
 			LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+				{
+					Key:      leaderworkerset.SetNameLabelKey,
+					Operator: metav1.LabelSelectorOpIn,
+					Values:   []string{lwsName},
+				},
 				{
 					Key:      podAffinityKey,
 					Operator: metav1.LabelSelectorOpExists,
diff --git a/pkg/webhooks/pod_webhook_test.go b/pkg/webhooks/pod_webhook_test.go
index af404fa85..e2f737b70 100644
--- a/pkg/webhooks/pod_webhook_test.go
+++ b/pkg/webhooks/pod_webhook_test.go
@@ -77,6 +77,7 @@ func TestSetExclusiveAffinities(t *testing.T) {
 			pod: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{"leaderworkerset.sigs.k8s.io/exclusive-topology": "topologyKey"},
+					Labels:      map[string]string{"leaderworkerset.sigs.k8s.io/name": "test-lws"},
 				},
 			},
 			groupUniqueKey: "test-key",
@@ -85,6 +86,7 @@ func TestSetExclusiveAffinities(t *testing.T) {
 			expectedPod: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{"leaderworkerset.sigs.k8s.io/exclusive-topology": "topologyKey"},
+					Labels:      map[string]string{"leaderworkerset.sigs.k8s.io/name": "test-lws"},
 				},
 				Spec: corev1.PodSpec{
 					Affinity: &corev1.Affinity{
@@ -104,6 +106,11 @@ func TestSetExclusiveAffinities(t *testing.T) {
 							RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{
 								TopologyKey: "topologyKey",
 								LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{
+									{
+										Key:      "leaderworkerset.sigs.k8s.io/name",
+										Operator: "In",
+										Values:   []string{"test-lws"},
+									},
 									{
 										Key:      "leaderworkerset.sigs.k8s.io/group-key",
 										Operator: "Exists",
@@ -125,6 +132,7 @@ func TestSetExclusiveAffinities(t *testing.T) {
 			pod: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{"leaderworkerset.sigs.k8s.io/exclusive-topology": "topologyKey"},
+					Labels:      map[string]string{"leaderworkerset.sigs.k8s.io/name": "test-lws"},
 				},
 				Spec: corev1.PodSpec{
 					Affinity: &corev1.Affinity{
@@ -143,6 +151,7 @@ func TestSetExclusiveAffinities(t *testing.T) {
 			expectedPod: &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{"leaderworkerset.sigs.k8s.io/exclusive-topology": "topologyKey"},
+					Labels:      map[string]string{"leaderworkerset.sigs.k8s.io/name": "test-lws"},
 				},
 				Spec: corev1.PodSpec{
 					Affinity: &corev1.Affinity{