diff --git a/operator/internal/controller/opsModes.go b/operator/internal/controller/opsModes.go index 2cbf5a15..4eeff3e9 100644 --- a/operator/internal/controller/opsModes.go +++ b/operator/internal/controller/opsModes.go @@ -55,6 +55,7 @@ func (r *ElastiServiceReconciler) switchMode(ctx context.Context, req ctrl.Reque return nil } +// TODO: Add atomicity to this, all-or-nothing func (r *ElastiServiceReconciler) enableProxyMode(ctx context.Context, req ctrl.Request, es *v1alpha1.ElastiService) error { targetNamespacedName := types.NamespacedName{ Name: es.Spec.Service, @@ -70,16 +71,16 @@ func (r *ElastiServiceReconciler) enableProxyMode(ctx context.Context, req ctrl. } r.Logger.Info("1. Checked and created private service", zap.String("public service", targetSVC.Name), zap.String("private service", PVTName)) + if err = r.createOrUpdateEndpointsliceToResolver(ctx, targetSVC); err != nil { + return fmt.Errorf("failed to create or update endpointslice to resolver: %w ", err) + } + r.Logger.Info("2. Created or updated endpointslice to resolver", zap.String("service", targetSVC.Name)) + // Check if Public Service is present, and has not changed from the values in CRDDirectory if err := r.watchPublicService(ctx, es, req); err != nil { return fmt.Errorf("failed to add watch on public service: %w", err) } - r.Logger.Info("2. Added watch on public service", zap.String("service", targetSVC.Name)) - - if err = r.createOrUpdateEndpointsliceToResolver(ctx, targetSVC); err != nil { - return fmt.Errorf("failed to create or update endpointslice to resolver: %w ", err) - } - r.Logger.Info("3. Created or updated endpointslice to resolver", zap.String("service", targetSVC.Name)) + r.Logger.Info("3. Added watch on public service", zap.String("service", targetSVC.Name)) return nil } diff --git a/tests/e2e/manifest/send_traffic.sh b/tests/e2e/manifest/send_traffic.sh index f8008c80..8eecf45c 100644 --- a/tests/e2e/manifest/send_traffic.sh +++ b/tests/e2e/manifest/send_traffic.sh @@ -61,6 +61,8 @@ NC='\033[0m' # No Color # log_failure_details: Centralized function to log detailed debugging information on failure log_failure_details() { + start_time="${1}" + echo "${RED}--- DETAILED FAILURE ANALYSIS ---${NC}" # Target EndpointSlice Status @@ -69,24 +71,24 @@ log_failure_details() { # Ingress Logs echo "${CYAN} Logs from ingress:${NC}" - kubectl logs -n istio-system deployments/istiod --tail=15 | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve resolver logs${NC}" + kubectl logs -n istio-system deployments/istiod --since-time="${start_time}" | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve ingress logs${NC}" # Resolver Logs echo "${CYAN} Logs from elasti-resolver:${NC}" - kubectl logs -n elasti services/elasti-resolver-service --tail=20 | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve resolver logs${NC}" + kubectl logs -n elasti services/elasti-resolver-service --since-time="${start_time}" --all-pods=true | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve resolver logs${NC}" # Controller Logs echo "${CYAN} Logs from elasti-controller:${NC}" - kubectl logs -n elasti services/elasti-operator-controller-service --tail=20 | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve controller logs${NC}" + kubectl logs -n elasti services/elasti-operator-controller-service --since-time="${start_time}" | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve controller logs${NC}" # Target Logs echo "${CYAN} Logs from target (${TARGET_RESOURCE}/${TARGET_NAME}):${NC}" - kubectl logs -n "$TARGET_NAMESPACE" "${TARGET_RESOURCE}/${TARGET_NAME}" --tail=120 | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve target pod logs${NC}" + kubectl logs -n "$TARGET_NAMESPACE" "${TARGET_RESOURCE}/${TARGET_NAME}" --since-time="${start_time}" | sed 's/^/ /' || echo "${YELLOW} - Could not retrieve target pod logs${NC}" # Verbose Curl Request echo "${CYAN} Attempting verbose request for more details...${NC}" kubectl exec -n "$CURL_NAMESPACE" "$CURL_POD_NAME" -- curl --max-time 10 -v -s "$URL" 2>&1 | head -20 | sed 's/^/ /' || echo "${YELLOW} - Verbose request failed${NC}" - + echo "${RED}-----------------------------------${NC}" } @@ -121,13 +123,20 @@ echo "${CYAN}--- Starting Traffic Test ---${NC}" failure_count=0 for i in $(seq 1 $MAX_RETRIES); do + if [ $i -lt 2 ]; then + sleep 5 + fi + echo "\n${CYAN}--- Request $i/$MAX_RETRIES ---${NC}" echo " ${CYAN}Time:${NC} $(date)" - echo " ${CYAN}Executing: kubectl exec -n $CURL_NAMESPACE $CURL_POD_NAME -- curl --max-time $TIMEOUT -s -o /dev/null -w \"%{http_code}\" \"$URL\"" + echo " ${CYAN}Executing: kubectl exec -n $CURL_NAMESPACE $CURL_POD_NAME -- curl --max-time $TIMEOUT --retry 2 --retry-delay 1 -s -o /dev/null -w \"%{http_code}\" \"$URL\"" start_time=$(date +%s) + start_time_rfc=$(date -u +"%Y-%m-%dT%H:%M:%SZ") code=$(kubectl exec -n "$CURL_NAMESPACE" "$CURL_POD_NAME" -- curl \ --max-time "$TIMEOUT" \ + --retry 2 \ + --retry-delay 1 \ -s \ -o /dev/null \ -w "%{http_code}" \ @@ -135,11 +144,11 @@ for i in $(seq 1 $MAX_RETRIES); do result=$? end_time=$(date +%s) duration=$((end_time - start_time)) - + echo " ${CYAN}Curl exit code:${NC} $result" echo " ${CYAN}HTTP status code:${NC} $code" echo " ${CYAN}Request duration:${NC} ${duration}s" - + # Detailed error analysis if [ "$result" != "0" ]; then echo "ERROR: Curl command failed with exit code $result" @@ -153,11 +162,11 @@ for i in $(seq 1 $MAX_RETRIES); do *) echo " - Unknown curl error" ;; esac - log_failure_details + log_failure_details "${start_time_rfc}" failure_count=$((failure_count + 1)) continue fi - + if [ "$code" != "200" ]; then echo "ERROR: Expected HTTP 200, got $code" case $code in @@ -170,13 +179,13 @@ for i in $(seq 1 $MAX_RETRIES); do *) echo " - HTTP error response" ;; esac - log_failure_details + log_failure_details "${start_time_rfc}" failure_count=$((failure_count + 1)) continue fi echo "${GREEN}SUCCESS: Request $i completed successfully (HTTP $code in ${duration}s)${NC}" - + if [ $i -lt $MAX_RETRIES ]; then sleep 1 fi diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/02-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/02-assert.yaml new file mode 100644 index 00000000..08bf1005 --- /dev/null +++ b/tests/e2e/tests/02-enable-serve-via-traffic/02-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: elasti.truefoundry.com/v1alpha1 +kind: ElastiService +metadata: + name: target-elastiservice +status: + mode: proxy diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/02-send-traffic-large.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/03-send-traffic-large.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/02-send-traffic-large.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/03-send-traffic-large.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/03-wait-ready.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/04-wait-ready.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/03-wait-ready.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/04-wait-ready.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/04-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/05-assert.yaml similarity index 52% rename from tests/e2e/tests/02-enable-serve-via-traffic/04-assert.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/05-assert.yaml index 94358076..af6897a1 100644 --- a/tests/e2e/tests/02-enable-serve-via-traffic/04-assert.yaml +++ b/tests/e2e/tests/02-enable-serve-via-traffic/05-assert.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: serve ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/05-scale-to-0-target.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/06-scale-to-0-target.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/05-scale-to-0-target.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/06-scale-to-0-target.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/07-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/07-assert.yaml new file mode 100644 index 00000000..08bf1005 --- /dev/null +++ b/tests/e2e/tests/02-enable-serve-via-traffic/07-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: elasti.truefoundry.com/v1alpha1 +kind: ElastiService +metadata: + name: target-elastiservice +status: + mode: proxy diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/06-send-traffic-drip.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/08-send-traffic-drip.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/06-send-traffic-drip.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/08-send-traffic-drip.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/07-wait-ready.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/09-wait-ready.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/07-wait-ready.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/09-wait-ready.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/08-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/10-assert.yaml similarity index 52% rename from tests/e2e/tests/02-enable-serve-via-traffic/08-assert.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/10-assert.yaml index 94358076..af6897a1 100644 --- a/tests/e2e/tests/02-enable-serve-via-traffic/08-assert.yaml +++ b/tests/e2e/tests/02-enable-serve-via-traffic/10-assert.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: serve ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/09-scale-to-0-target.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/11-scale-to-0-target.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/09-scale-to-0-target.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/11-scale-to-0-target.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/12-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/12-assert.yaml index 94358076..08bf1005 100644 --- a/tests/e2e/tests/02-enable-serve-via-traffic/12-assert.yaml +++ b/tests/e2e/tests/02-enable-serve-via-traffic/12-assert.yaml @@ -3,9 +3,4 @@ kind: ElastiService metadata: name: target-elastiservice status: - mode: serve ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 + mode: proxy diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/10-send-traffic-stream.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/13-send-traffic-stream.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/10-send-traffic-stream.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/13-send-traffic-stream.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/11-wait-ready.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/14-wait-ready.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/11-wait-ready.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/14-wait-ready.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/16-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/15-assert.yaml similarity index 52% rename from tests/e2e/tests/02-enable-serve-via-traffic/16-assert.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/15-assert.yaml index 94358076..af6897a1 100644 --- a/tests/e2e/tests/02-enable-serve-via-traffic/16-assert.yaml +++ b/tests/e2e/tests/02-enable-serve-via-traffic/15-assert.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: serve ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/13-scale-to-0-target.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/16-scale-to-0-target.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/13-scale-to-0-target.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/16-scale-to-0-target.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/17-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/17-assert.yaml new file mode 100644 index 00000000..08bf1005 --- /dev/null +++ b/tests/e2e/tests/02-enable-serve-via-traffic/17-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: elasti.truefoundry.com/v1alpha1 +kind: ElastiService +metadata: + name: target-elastiservice +status: + mode: proxy diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/14-send-traffic-small.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/18-send-traffic-small.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/14-send-traffic-small.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/18-send-traffic-small.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/15-wait-ready.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/19-wait-ready.yaml similarity index 100% rename from tests/e2e/tests/02-enable-serve-via-traffic/15-wait-ready.yaml rename to tests/e2e/tests/02-enable-serve-via-traffic/19-wait-ready.yaml diff --git a/tests/e2e/tests/02-enable-serve-via-traffic/20-assert.yaml b/tests/e2e/tests/02-enable-serve-via-traffic/20-assert.yaml new file mode 100644 index 00000000..af6897a1 --- /dev/null +++ b/tests/e2e/tests/02-enable-serve-via-traffic/20-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: elasti.truefoundry.com/v1alpha1 +kind: ElastiService +metadata: + name: target-elastiservice +status: + mode: serve diff --git a/tests/e2e/tests/03-enable-serve-via-manual/03-assert.yaml b/tests/e2e/tests/03-enable-serve-via-manual/03-assert.yaml index 94358076..af6897a1 100644 --- a/tests/e2e/tests/03-enable-serve-via-manual/03-assert.yaml +++ b/tests/e2e/tests/03-enable-serve-via-manual/03-assert.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: serve ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/06-resolver-endpoint-sync/06-scale-to-1-resolver.yaml b/tests/e2e/tests/06-resolver-endpoint-sync/06-scale-to-1-resolver.yaml new file mode 100644 index 00000000..d5050284 --- /dev/null +++ b/tests/e2e/tests/06-resolver-endpoint-sync/06-scale-to-1-resolver.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl scale deployment/elasti-resolver -n elasti --replicas=1 + - command: kubectl rollout status deployment/elasti-resolver -n elasti --timeout=400s diff --git a/tests/e2e/tests/07-scale-down-with-filter/05-assert.yaml b/tests/e2e/tests/07-scale-down-with-filter/05-assert.yaml index e2f7b6c2..08bf1005 100644 --- a/tests/e2e/tests/07-scale-down-with-filter/05-assert.yaml +++ b/tests/e2e/tests/07-scale-down-with-filter/05-assert.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: proxy ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/08-resolver-multiple-endpoints/06-assert-mode-proxy.yaml b/tests/e2e/tests/08-resolver-multiple-endpoints/06-assert-mode-proxy.yaml index e2f7b6c2..08bf1005 100644 --- a/tests/e2e/tests/08-resolver-multiple-endpoints/06-assert-mode-proxy.yaml +++ b/tests/e2e/tests/08-resolver-multiple-endpoints/06-assert-mode-proxy.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: proxy ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/08-resolver-multiple-endpoints/08-assert-mode-serve.yaml b/tests/e2e/tests/08-resolver-multiple-endpoints/08-assert-mode-serve.yaml index 94358076..af6897a1 100644 --- a/tests/e2e/tests/08-resolver-multiple-endpoints/08-assert-mode-serve.yaml +++ b/tests/e2e/tests/08-resolver-multiple-endpoints/08-assert-mode-serve.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: serve ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/09-statefulset-serve-via-traffic/02-assert.yaml b/tests/e2e/tests/09-statefulset-serve-via-traffic/02-assert.yaml index e2f7b6c2..08bf1005 100644 --- a/tests/e2e/tests/09-statefulset-serve-via-traffic/02-assert.yaml +++ b/tests/e2e/tests/09-statefulset-serve-via-traffic/02-assert.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: proxy ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60 diff --git a/tests/e2e/tests/09-statefulset-serve-via-traffic/05-assert.yaml b/tests/e2e/tests/09-statefulset-serve-via-traffic/05-assert.yaml index 94358076..af6897a1 100644 --- a/tests/e2e/tests/09-statefulset-serve-via-traffic/05-assert.yaml +++ b/tests/e2e/tests/09-statefulset-serve-via-traffic/05-assert.yaml @@ -4,8 +4,3 @@ metadata: name: target-elastiservice status: mode: serve ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -# give operator time — 60s is reasonable here -timeout: 60