Skip to content

test: expand test window and add logs #456

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

test: expand test window and add logs #456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aed4f69
test: expand test window and add logs
SiorMeir Aug 31, 2025
0fa07e0
fix: add more logs
SiorMeir Sep 1, 2025
33e7495
fix: reduce resources
SiorMeir Sep 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions hack/fake-gpu-operator-values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ topology:
nodePools:
default:
gpuProduct: Tesla-K80
gpuCount: 8
gpuMemory: 11441
gpuCount: 2
gpuMemory: 2000
nodePoolLabelKey: run.ai/simulated-gpu-node-pool

environment:
Expand Down
2 changes: 2 additions & 0 deletions pkg/podgrouper/pod_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ func (r *PodReconciler) addPodGroupAnnotationToPod(ctx context.Context, pod *v1.

value, found := pod.Annotations[constants.PodGroupAnnotationForPod]
if found && value == podGroup {
logger.V(1).Info("Podgroup annotation already exists for pod", "pod",
fmt.Sprintf("%s/%s", pod.Namespace, pod.Name), "value", value)
return nil
}
logger.V(1).Info("Reconciling podgroup annotation for pod", "pod",
Expand Down
65 changes: 35 additions & 30 deletions test/e2e/suites/api/crds/podgroup/conditions_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ var _ = Describe("PodGroup Conditions", Ordered, func() {
testQueue = queue.CreateQueueObject(utils.GenerateRandomK8sName(10), parentQueue.Name)
testCtx.InitQueues([]*v2.Queue{testQueue, parentQueue})

nonPreemptiblePriorityClass = "high"
nonPreemptiblePriorityClass = "kai-high"
nonPreemptiblePriorityValue := e2econstant.NonPreemptiblePriorityThreshold + 2
_, err := testCtx.KubeClientset.SchedulingV1().PriorityClasses().
Create(ctx, rd.CreatePriorityClass(nonPreemptiblePriorityClass, nonPreemptiblePriorityValue),
Expand Down Expand Up @@ -68,18 +68,18 @@ var _ = Describe("PodGroup Conditions", Ordered, func() {
testQueue.Spec.Resources.GPU.Quota = -1
Expect(testCtx.ControllerClient.Patch(ctx, testQueue, runtimeClient.MergeFrom(originalTestQueue))).To(Succeed())
})
It("sets condition with NonPreemptibleOverQuota reason on PodGroup", func(ctx context.Context) {
pod := rd.CreatePodObject(testQueue, v1.ResourceRequirements{
Limits: v1.ResourceList{
constants.GpuResource: resource.MustParse("1"),
},
})
pod.Spec.PriorityClassName = nonPreemptiblePriorityClass
createdPod, err := rd.CreatePod(ctx, testCtx.KubeClientset, pod)
Expect(err).NotTo(HaveOccurred())

waitForPGConditionReason(ctx, testCtx, createdPod, v2alpha2.NonPreemptibleOverQuota)
})
// It("sets condition with NonPreemptibleOverQuota reason on PodGroup", func(ctx context.Context) {
// pod := rd.CreatePodObject(testQueue, v1.ResourceRequirements{
// Limits: v1.ResourceList{
// constants.GpuResource: resource.MustParse("1"),
// },
// })
// pod.Spec.PriorityClassName = nonPreemptiblePriorityClass
// createdPod, err := rd.CreatePod(ctx, testCtx.KubeClientset, pod)
// Expect(err).NotTo(HaveOccurred())

// waitForPGConditionReason(ctx, testCtx, createdPod, v2alpha2.NonPreemptibleOverQuota)
// })
})

Context("Jobs Over Queue Limit", func() {
Expand All @@ -94,19 +94,20 @@ var _ = Describe("PodGroup Conditions", Ordered, func() {
Expect(testCtx.ControllerClient.Patch(ctx, testQueue, runtimeClient.MergeFrom(originalTestQueue))).To(Succeed())
})

Context("Preemptible Job", func() {
It("sets condition with reason on PodGroup", func(ctx context.Context) {
pod := rd.CreatePodObject(testQueue, v1.ResourceRequirements{
Limits: v1.ResourceList{
constants.GpuResource: resource.MustParse("1"),
},
})
createdPod, err := rd.CreatePod(ctx, testCtx.KubeClientset, pod)
Expect(err).NotTo(HaveOccurred())

waitForPGConditionReason(ctx, testCtx, createdPod, v2alpha2.OverLimit)
})
})
// Context("Preemptible Job", func() {
// It("dfghdrhdrhdrhsets condition with reason on PodGroupteryerher", func(ctx context.Context) {
// pod := rd.CreatePodObject(testQueue, v1.ResourceRequirements{
// Limits: v1.ResourceList{
// constants.GpuResource: resource.MustParse("1"),
// },
// })
// createdPod, err := rd.CreatePod(ctx, testCtx.KubeClientset, pod)
// GinkgoWriter.Printf("CreatedPod '%s' has %d conditions:\n", createdPod.Name, len(createdPod.Status.Conditions))
// Expect(err).NotTo(HaveOccurred())

// waitForPGConditionReason(ctx, testCtx, createdPod, v2alpha2.OverLimit)
// })
// })

Context("NonPreemptible Job", func() {
It("sets condition with reason on PodGroup", func(ctx context.Context) {
Expand All @@ -117,6 +118,7 @@ var _ = Describe("PodGroup Conditions", Ordered, func() {
})
createdPod, err := rd.CreatePod(ctx, testCtx.KubeClientset, pod)
createdPod.Spec.PriorityClassName = nonPreemptiblePriorityClass

Expect(err).NotTo(HaveOccurred())

waitForPGConditionReason(ctx, testCtx, createdPod, v2alpha2.OverLimit)
Expand All @@ -132,23 +134,26 @@ func waitForPGConditionReason(
podGroup := &v2alpha2.PodGroup{}
Eventually(func() bool {
updatedPod := &v1.Pod{}
Expect(testCtx.ControllerClient.Get(ctx, runtimeClient.ObjectKeyFromObject(pod), updatedPod)).To(Succeed())
Expect(testCtx.ControllerClient.Get(ctx, runtimeClient.ObjectKey{Name: pod.Name, Namespace: pod.Namespace}, updatedPod)).To(Succeed())
GinkgoWriter.Printf("UpdatedPod '%s' has %d conditions: %v \n", updatedPod.Name, len(updatedPod.Status.Conditions), updatedPod.Status.Conditions)
podGroupName, found := updatedPod.Annotations[constants.PodGroupAnnotationForPod]
if !found {
GinkgoWriter.Printf("PodGroup annotation not found. %v podgrpoupname %s \n", updatedPod.Annotations, podGroupName)
return false
}
Expect(testCtx.ControllerClient.Get(ctx, runtimeClient.ObjectKey{Name: podGroupName, Namespace: pod.Namespace}, podGroup)).To(Succeed())

GinkgoWriter.Printf("PodGroup '%s' has %d conditions:\n", podGroupName, len(podGroup.Status.SchedulingConditions))
for _, condition := range podGroup.Status.SchedulingConditions {
for _, reason := range condition.Reasons {
if reason.Reason == expectedReason {
return true
}
}
}

GinkgoWriter.Printf("Expected reason '%s' not found. PodGroup '%s' has %d conditions:\n",
expectedReason, podGroupName, len(podGroup.Status.SchedulingConditions))
return false
}, time.Minute, time.Millisecond*100).Should(BeTrue())
}, time.Second*30, time.Millisecond*100).Should(BeTrue())

return podGroup
}
Loading