diff --git a/xset/api/opslifecycle_types.go b/xset/api/v1alpha1/opslifecycle_types.go similarity index 57% rename from xset/api/opslifecycle_types.go rename to xset/api/v1alpha1/opslifecycle_types.go index 78ac10c..e56863c 100644 --- a/xset/api/opslifecycle_types.go +++ b/xset/api/v1alpha1/opslifecycle_types.go @@ -14,46 +14,12 @@ * limitations under the License. */ -package api +package v1alpha1 import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type OperationLabelEnum int - -const ( - // OperatingLabelPrefix indicates a target is under operation - // set by xset controller - OperatingLabelPrefix OperationLabelEnum = iota - - // OperationTypeLabelPrefix indicates the type of operation - // set by xset controller - OperationTypeLabelPrefix - - // OperateLabelPrefix indicates a target could start operation - // set by related opsLifecycle controller. - // xset controller will start operation only after this label is set - OperateLabelPrefix - - // UndoOperationTypeLabelPrefix indicates a type of operation has been canceled. - // need to be handled by related opsLifecycle controller - UndoOperationTypeLabelPrefix - - // ServiceAvailableLabel indicates a target is available for service. - // set by related opsLifecycle controller. - ServiceAvailableLabel - - // PreparingDeleteLabel indicates a target is preparing to be deleted. - // set by xset controller, - // handle by related opsLifecycle controller if needed. - PreparingDeleteLabel -) - -type LifeCycleLabelManager interface { - Get(labelType OperationLabelEnum) string -} - type OperationType string var ( diff --git a/xset/api/resourcecontext_types.go b/xset/api/v1alpha1/resourcecontext_types.go similarity index 95% rename from xset/api/resourcecontext_types.go rename to xset/api/v1alpha1/resourcecontext_types.go index 67f3b5e..5e51ca9 100644 --- a/xset/api/resourcecontext_types.go +++ b/xset/api/v1alpha1/resourcecontext_types.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package api +package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,7 +37,7 @@ type ResourceContextAdapter interface { // ResourceContextKeyEnum defines the key of resource context type ResourceContextKeyEnum int -const EnumContextKeyNum = 6 +const EnumContextKeyNum = 8 const ( EnumOwnerContextKey ResourceContextKeyEnum = iota EnumRevisionContextDataKey @@ -45,6 +45,8 @@ const ( EnumJustCreateContextDataKey EnumRecreateUpdateContextDataKey EnumScaleInContextDataKey + EnumReplaceNewTargetIDContextDataKey + EnumReplaceOriginTargetIDContextDataKey ) // ResourceContextSpec defines the desired state of ResourceContext diff --git a/xset/api/validation/resourcecontext.go b/xset/api/v1alpha1/validation/resourcecontext.go similarity index 53% rename from xset/api/validation/resourcecontext.go rename to xset/api/v1alpha1/validation/resourcecontext.go index dd9708c..dc038e5 100644 --- a/xset/api/validation/resourcecontext.go +++ b/xset/api/v1alpha1/validation/resourcecontext.go @@ -19,38 +19,28 @@ package validation import ( "errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) // ValidateResourceContextAdapter validates the resource context adapter -func ValidateResourceContextAdapter(adapter api.ResourceContextAdapter) error { +func ValidateResourceContextAdapter(adapter v1alpha1.ResourceContextAdapter) error { if adapter == nil { return errors.New("resource context adapter is nil") } - var errList []error - errList = append(errList, - validateResourceContextMeta(adapter.ResourceContextMeta()), - validateResourceContextKey(adapter.GetContextKeys())) - return errors.Join(errList...) -} - -func validateResourceContextMeta(t metav1.TypeMeta) error { - if len(t.String()) == 0 { - return errors.New("resource context meta is not valid") - } - return nil + return errors.Join( + validateMeta(adapter.ResourceContextMeta()), + validateResourceContextKey(adapter.GetContextKeys()), + ) } -func validateResourceContextKey(m map[api.ResourceContextKeyEnum]string) error { +func validateResourceContextKey(m map[v1alpha1.ResourceContextKeyEnum]string) error { if m == nil { return errors.New("resource context keys is nil") } - for i := range api.EnumContextKeyNum { - if _, ok := m[api.ResourceContextKeyEnum(i)]; !ok { - return errors.New("resource context keys is not valid, please add enough context keys") + for i := range v1alpha1.EnumContextKeyNum { + if _, ok := m[v1alpha1.ResourceContextKeyEnum(i)]; !ok { + return errors.New("resource context keys are not valid, please add enough context keys") } } return nil diff --git a/xset/api/v1alpha1/validation/shared.go b/xset/api/v1alpha1/validation/shared.go new file mode 100644 index 0000000..7d75dea --- /dev/null +++ b/xset/api/v1alpha1/validation/shared.go @@ -0,0 +1,30 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "errors" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func validateMeta(t metav1.TypeMeta) error { + if len(t.String()) == 0 { + return errors.New("resource context meta is not valid") + } + return nil +} diff --git a/xset/api/v1alpha1/validation/xset_controller.go b/xset/api/v1alpha1/validation/xset_controller.go new file mode 100644 index 0000000..d04e4a3 --- /dev/null +++ b/xset/api/v1alpha1/validation/xset_controller.go @@ -0,0 +1,61 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/util/validation" + + "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" +) + +// ValidateXSetController validates the XSetController +func ValidateXSetController(xSetController v1alpha1.XSetController) error { + if xSetController == nil { + return errors.New("xSetController is nil") + } + return errors.Join( + validateMeta(xSetController.XSetMeta()), + validateMeta(xSetController.XMeta()), + validateFinalizerName(xSetController.FinalizerName()), + validateXSetLabelAnnotationManager(xSetController.GetXSetControllerLabelManager()), + ) +} + +func validateFinalizerName(name string) error { + msg := validation.IsQualifiedName(name) + if len(msg) > 0 { + return errors.New(fmt.Sprintf("%v", msg)) + } + return nil +} + +func validateXSetLabelAnnotationManager(manager api.XSetLabelAnnotationManager) error { + if manager == nil { + return nil + } + + for i := range api.EnumXSetLabelAnnotationsNum { + if len(manager.Value(api.XSetLabelAnnotationEnum(i))) == 0 { + return errors.New("XSetLabelAnnotationManager label annotations are not valid, please add enough label annotations") + } + } + return nil +} diff --git a/xset/api/xset_controller_types.go b/xset/api/v1alpha1/xset_controller_types.go similarity index 76% rename from xset/api/xset_controller_types.go rename to xset/api/v1alpha1/xset_controller_types.go index 983bfd0..907e727 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/v1alpha1/xset_controller_types.go @@ -14,12 +14,16 @@ * limitations under the License. */ -package api +package v1alpha1 import ( + "context" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/xset/api" ) type XSetController interface { @@ -35,11 +39,12 @@ type XSetController interface { GetXSetSpec(object XSetObject) *XSetSpec GetXSetPatch(object metav1.Object) ([]byte, error) - UpdateScaleStrategy(object XSetObject, scaleStrategy *ScaleStrategy) (err error) + GetXSetTemplatePatcher(object metav1.Object) func(client.Object) error GetXSetStatus(object XSetObject) *XSetStatus SetXSetStatus(object XSetObject, status *XSetStatus) + GetReadyTime(object client.Object) *metav1.Time - GetLifeCycleLabelManager() LifeCycleLabelManager + GetXSetControllerLabelManager() api.XSetLabelAnnotationManager GetScaleInOpsLifecycleAdapter() LifecycleAdapter GetUpdateOpsLifecycleAdapter() LifecycleAdapter GetResourceContextAdapter() ResourceContextAdapter @@ -47,6 +52,9 @@ type XSetController interface { CheckScheduled(object client.Object) bool CheckReady(object client.Object) bool CheckAvailable(object client.Object) bool + + UpdateScaleStrategy(ctx context.Context, c client.Client, object XSetObject, scaleStrategy *ScaleStrategy) error + GetXOpsPriority(ctx context.Context, c client.Client, object client.Object) (*OpsPriority, error) } type XSetObject client.Object diff --git a/xset/api/xset_types.go b/xset/api/v1alpha1/xset_types.go similarity index 95% rename from xset/api/xset_types.go rename to xset/api/v1alpha1/xset_types.go index ff8056d..98e2f03 100644 --- a/xset/api/xset_types.go +++ b/xset/api/v1alpha1/xset_types.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package api +package v1alpha1 // +k8s:deepcopy-gen=file @@ -188,7 +188,7 @@ type XSetStatus struct { AvailableReplicas int32 `json:"availableReplicas,omitempty"` // UpdatedAvailableReplicas indicates the number of available updated revision replicas for this replicas set. - // A model is updated available means the model is ready for updated revision and accessible + // A target is updated available means the target is ready for updated revision and accessible // +optional UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"` @@ -196,3 +196,11 @@ type XSetStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` } + +// OpsPriority is used to store the ops priority of a target +type OpsPriority struct { + // PriorityClass is the priority class of the target + PriorityClass int32 + // DeletionCost is the deletion cost of the target + DeletionCost int32 +} diff --git a/xset/api/zz_generated.deepcopy.go b/xset/api/v1alpha1/zz_generated.deepcopy.go similarity index 99% rename from xset/api/zz_generated.deepcopy.go rename to xset/api/v1alpha1/zz_generated.deepcopy.go index 46192a5..351c657 100644 --- a/xset/api/zz_generated.deepcopy.go +++ b/xset/api/v1alpha1/zz_generated.deepcopy.go @@ -18,7 +18,7 @@ // Code generated by controller-gen. DO NOT EDIT. -package api +package v1alpha1 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/xset/api/well_knowns.go b/xset/api/well_knowns.go new file mode 100644 index 0000000..27ad7ac --- /dev/null +++ b/xset/api/well_knowns.go @@ -0,0 +1,169 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api + +import appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + +type XSetLabelAnnotationEnum int + +const EnumXSetLabelAnnotationsNum = 19 +const ( + // OperatingLabelPrefix indicates a target is under operation + // set by xset controller + OperatingLabelPrefix XSetLabelAnnotationEnum = iota + + // OperationTypeLabelPrefix indicates the type of operation + // set by xset controller + OperationTypeLabelPrefix + + // OperateLabelPrefix indicates a target could start operation + // set by related opsLifecycle controller. + // xset controller will start operation only after this label is set + OperateLabelPrefix + + // UndoOperationTypeLabelPrefix indicates a type of operation has been canceled. + // need to be handled by related opsLifecycle controller + UndoOperationTypeLabelPrefix + + // ServiceAvailableLabel indicates a target is available for service. + // set by related opsLifecycle controller. + ServiceAvailableLabel + + // PreparingDeleteLabel indicates a target is preparing to be deleted. + // set by xset controller, + // handle by related opsLifecycle controller if needed. + PreparingDeleteLabel + + // ControlledByXSetLabel indicates a target is controlled by xset. + // set by xset controller + ControlledByXSetLabel + + // XInstanceIdLabelKey is used to attach instance ID on x + XInstanceIdLabelKey + + // XSetUpdateIndicationLabelKey is used to indicate a target is updated by xset + XSetUpdateIndicationLabelKey + + // XDeletionIndicationLabelKey is used to indicate a target is deleted by xset + XDeletionIndicationLabelKey + + // XReplaceIndicationLabelKey is used to indicate a target is replaced by xset + XReplaceIndicationLabelKey + + // XReplacePairNewId is used to indicate the new created target on replace origin target + XReplacePairNewId + + // XReplacePairOriginName is used to indicate replace origin target name on the new created target + XReplacePairOriginName + + // XReplaceByReplaceUpdateLabelKey indicates a target is replaced by update by xset + XReplaceByReplaceUpdateLabelKey + + // XOrphanedIndicationLabelKey is used to indicate a target is orphaned by xset + XOrphanedIndicationLabelKey + + // XCreatingLabel indicates a target is creating by xset + XCreatingLabel + + // XCompletingLabel indicates a target is completing by xset + XCompletingLabel + + // XExcludeIndicationLabelKey is used to indicate a target is excluded by xset + XExcludeIndicationLabelKey + + // LastXStatusAnnotationKey is used to record the last status of a target by xset + LastXStatusAnnotationKey +) + +type XSetLabelAnnotationManager interface { + Get(labels map[string]string, labelType XSetLabelAnnotationEnum) (string, bool) + Set(labels map[string]string, labelType XSetLabelAnnotationEnum, value string) + Delete(labels map[string]string, labelType XSetLabelAnnotationEnum) + Value(labelType XSetLabelAnnotationEnum) string +} + +var defaultXSetLabelAnnotationManager = map[XSetLabelAnnotationEnum]string{ + OperatingLabelPrefix: appsv1alpha1.PodOperatingLabelPrefix, + OperationTypeLabelPrefix: appsv1alpha1.PodOperationTypeLabelPrefix, + OperateLabelPrefix: appsv1alpha1.PodOperateLabelPrefix, + UndoOperationTypeLabelPrefix: appsv1alpha1.PodUndoOperationTypeLabelPrefix, + ServiceAvailableLabel: appsv1alpha1.PodServiceAvailableLabel, + PreparingDeleteLabel: appsv1alpha1.PodPreparingDeleteLabel, + + ControlledByXSetLabel: appsv1alpha1.ControlledByKusionStackLabelKey, + XInstanceIdLabelKey: appsv1alpha1.PodInstanceIDLabelKey, + XSetUpdateIndicationLabelKey: appsv1alpha1.CollaSetUpdateIndicateLabelKey, + XDeletionIndicationLabelKey: appsv1alpha1.PodDeletionIndicationLabelKey, + XReplaceIndicationLabelKey: appsv1alpha1.PodReplaceIndicationLabelKey, + XReplacePairNewId: appsv1alpha1.PodReplacePairNewId, + XReplacePairOriginName: appsv1alpha1.PodReplacePairOriginName, + XReplaceByReplaceUpdateLabelKey: appsv1alpha1.PodReplaceByReplaceUpdateLabelKey, + XOrphanedIndicationLabelKey: appsv1alpha1.PodOrphanedIndicateLabelKey, + XCreatingLabel: appsv1alpha1.PodCreatingLabel, + XCompletingLabel: appsv1alpha1.PodCompletingLabel, + XExcludeIndicationLabelKey: appsv1alpha1.PodExcludeIndicationLabelKey, + + LastXStatusAnnotationKey: appsv1alpha1.LastPodStatusAnnotationKey, +} + +func NewXSetLabelAnnotationManager() XSetLabelAnnotationManager { + return &xSetLabelAnnotationManager{ + labelManager: defaultXSetLabelAnnotationManager, + } +} + +type xSetLabelAnnotationManager struct { + labelManager map[XSetLabelAnnotationEnum]string +} + +func (m *xSetLabelAnnotationManager) Get(labels map[string]string, key XSetLabelAnnotationEnum) (string, bool) { + if labels == nil { + return "", false + } + labelKey := m.labelManager[key] + val, exist := labels[labelKey] + return val, exist +} + +func (m *xSetLabelAnnotationManager) Set(labels map[string]string, key XSetLabelAnnotationEnum, val string) { + if labels == nil { + labels = make(map[string]string) + } + labelKey := m.labelManager[key] + labels[labelKey] = val +} + +func (m *xSetLabelAnnotationManager) Delete(labels map[string]string, key XSetLabelAnnotationEnum) { + if labels == nil { + return + } + labelKey := m.labelManager[key] + delete(labels, labelKey) +} + +func (m *xSetLabelAnnotationManager) Value(key XSetLabelAnnotationEnum) string { + return m.labelManager[key] +} + +func GetWellKnownLabelPrefixesWithID(m XSetLabelAnnotationManager) []string { + return []string{ + m.Value(OperatingLabelPrefix), + m.Value(OperationTypeLabelPrefix), + m.Value(UndoOperationTypeLabelPrefix), + m.Value(OperatingLabelPrefix), + } +} diff --git a/xset/opslifecycle/default_adapters.go b/xset/opslifecycle/default_adapters.go index 323b646..88cfdaf 100644 --- a/xset/opslifecycle/default_adapters.go +++ b/xset/opslifecycle/default_adapters.go @@ -17,50 +17,56 @@ package opslifecycle import ( + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) -var _ api.LifecycleAdapter = &DefaultUpdateLifecycleAdapter{} +var _ v1alpha1.LifecycleAdapter = &DefaultUpdateLifecycleAdapter{} type DefaultUpdateLifecycleAdapter struct { - LabelManager api.LifeCycleLabelManager + LabelAnnoManager api.XSetLabelAnnotationManager + XSetType metav1.TypeMeta } func (d *DefaultUpdateLifecycleAdapter) GetID() string { - return "xset" + return strings.ToLower(d.XSetType.Kind) } -func (d *DefaultUpdateLifecycleAdapter) GetType() api.OperationType { - return api.OpsLifecycleTypeUpdate +func (d *DefaultUpdateLifecycleAdapter) GetType() v1alpha1.OperationType { + return v1alpha1.OpsLifecycleTypeUpdate } func (d *DefaultUpdateLifecycleAdapter) AllowMultiType() bool { return true } -func (d *DefaultUpdateLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { - setOperate(d.LabelManager, d, target) +func (d *DefaultUpdateLifecycleAdapter) WhenBegin(_ client.Object) (bool, error) { return true, nil } func (d *DefaultUpdateLifecycleAdapter) WhenFinish(target client.Object) (bool, error) { + // TODO inplace update post actions return false, nil } -var _ api.LifecycleAdapter = &DefaultScaleInLifecycleAdapter{} +var _ v1alpha1.LifecycleAdapter = &DefaultScaleInLifecycleAdapter{} type DefaultScaleInLifecycleAdapter struct { - LabelManager api.LifeCycleLabelManager + LabelAnnoManager api.XSetLabelAnnotationManager + XSetType metav1.TypeMeta } func (d *DefaultScaleInLifecycleAdapter) GetID() string { - return "xset" + return strings.ToLower(d.XSetType.Kind) } -func (d *DefaultScaleInLifecycleAdapter) GetType() api.OperationType { - return api.OpsLifecycleTypeScaleIn +func (d *DefaultScaleInLifecycleAdapter) GetType() v1alpha1.OperationType { + return v1alpha1.OpsLifecycleTypeScaleIn } func (d *DefaultScaleInLifecycleAdapter) AllowMultiType() bool { @@ -68,10 +74,9 @@ func (d *DefaultScaleInLifecycleAdapter) AllowMultiType() bool { } func (d *DefaultScaleInLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { - setOperate(d.LabelManager, d, target) - return true, nil + return WhenBeginDelete(d.LabelAnnoManager, target) } -func (d *DefaultScaleInLifecycleAdapter) WhenFinish(target client.Object) (bool, error) { +func (d *DefaultScaleInLifecycleAdapter) WhenFinish(_ client.Object) (bool, error) { return false, nil } diff --git a/xset/opslifecycle/label_manager.go b/xset/opslifecycle/label_manager.go deleted file mode 100644 index 0de7e3d..0000000 --- a/xset/opslifecycle/label_manager.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2024 - 2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package opslifecycle - -import ( - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - - "kusionstack.io/kube-utils/xset/api" -) - -// OpsLifecycle default labels -var ( - defaultOperatingLabelPrefix = appsv1alpha1.PodOperatingLabelPrefix - defaultOperationTypeLabelPrefix = appsv1alpha1.PodOperationTypeLabelPrefix - defaultOperateLabelPrefix = appsv1alpha1.PodOperateLabelPrefix - defaultUndoOperationTypeLabelPrefix = appsv1alpha1.PodUndoOperationTypeLabelPrefix - defaultServiceAvailableLabel = appsv1alpha1.PodServiceAvailableLabel - defaultPreparingDeleteLabel = appsv1alpha1.PodPreparingDeleteLabel -) - -var defaultLabels = map[api.OperationLabelEnum]string{ - api.OperatingLabelPrefix: defaultOperatingLabelPrefix, - api.OperationTypeLabelPrefix: defaultOperationTypeLabelPrefix, - api.OperateLabelPrefix: defaultOperateLabelPrefix, - api.UndoOperationTypeLabelPrefix: defaultUndoOperationTypeLabelPrefix, - api.ServiceAvailableLabel: defaultServiceAvailableLabel, - api.PreparingDeleteLabel: defaultPreparingDeleteLabel, -} - -type LabelManagerImpl struct { - labels map[api.OperationLabelEnum]string - wellKnownLabelPrefixesWithID []string -} - -func NewLabelManager(overwrite map[api.OperationLabelEnum]string) api.LifeCycleLabelManager { - labelKeys := make(map[api.OperationLabelEnum]string) - for k, v := range defaultLabels { - labelKeys[k] = v - } - if len(overwrite) > 0 { - for k, v := range overwrite { - labelKeys[k] = v - } - } - - wellKnownLabelPrefixesWithID := []string{ - labelKeys[api.OperatingLabelPrefix], - labelKeys[api.OperationTypeLabelPrefix], - labelKeys[api.UndoOperationTypeLabelPrefix], - labelKeys[api.OperatingLabelPrefix], - } - return &LabelManagerImpl{ - labels: labelKeys, - wellKnownLabelPrefixesWithID: wellKnownLabelPrefixesWithID, - } -} - -func (m *LabelManagerImpl) Get(labelType api.OperationLabelEnum) string { - return m.labels[labelType] -} diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go index 4f56a03..8e7a08e 100644 --- a/xset/opslifecycle/utils.go +++ b/xset/opslifecycle/utils.go @@ -27,25 +27,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) type UpdateFunc func(object client.Object) (bool, error) -func IsServiceAvailable(m api.LifeCycleLabelManager, target client.Object) bool { - _, exists := target.GetLabels()[m.Get(api.ServiceAvailableLabel)] - return exists -} - // IDToLabelsMap returns a map of pod id to labels map and a map of operation type to number of pods. -func IDToLabelsMap(m *LabelManagerImpl, target client.Object) (map[string]map[string]string, map[string]int, error) { +func IDToLabelsMap(m api.XSetLabelAnnotationManager, target client.Object) (map[string]map[string]string, map[string]int, error) { idToLabelsMap := map[string]map[string]string{} typeToNumsMap := map[string]int{} ids := sets.String{} labels := target.GetLabels() for k := range labels { - if strings.HasPrefix(k, m.Get(api.OperatingLabelPrefix)) || - strings.HasPrefix(k, m.Get(api.OperateLabelPrefix)) { + if strings.HasPrefix(k, m.Value(api.OperatingLabelPrefix)) || + strings.HasPrefix(k, m.Value(api.OperateLabelPrefix)) { s := strings.Split(k, "/") if len(s) < 2 { return nil, nil, fmt.Errorf("invalid label %s", k) @@ -55,11 +51,11 @@ func IDToLabelsMap(m *LabelManagerImpl, target client.Object) (map[string]map[st } for id := range ids { - if operationType, ok := labels[fmt.Sprintf("%s/%s", m.Get(api.OperationTypeLabelPrefix), id)]; ok { + if operationType, ok := labels[fmt.Sprintf("%s/%s", m.Value(api.OperationTypeLabelPrefix), id)]; ok { typeToNumsMap[operationType] += 1 } - for _, prefix := range m.wellKnownLabelPrefixesWithID { + for _, prefix := range api.GetWellKnownLabelPrefixesWithID(m) { label := fmt.Sprintf("%s/%s", prefix, id) value, ok := labels[label] if !ok { @@ -78,7 +74,7 @@ func IDToLabelsMap(m *LabelManagerImpl, target client.Object) (map[string]map[st } // NumOfLifecycleOnTarget returns the nums of lifecycles on pod -func NumOfLifecycleOnTarget(m *LabelManagerImpl, target client.Object) (int, error) { +func NumOfLifecycleOnTarget(m api.XSetLabelAnnotationManager, target client.Object) (int, error) { if target == nil { return 0, nil } @@ -86,8 +82,8 @@ func NumOfLifecycleOnTarget(m *LabelManagerImpl, target client.Object) (int, err return len(newIDToLabelsMap), err } -func WhenBeginDelete(m api.LifeCycleLabelManager, obj client.Object) (bool, error) { - return AddLabel(obj, m.Get(api.PreparingDeleteLabel), strconv.FormatInt(time.Now().UnixNano(), 10)), nil +func WhenBeginDelete(m api.XSetLabelAnnotationManager, obj client.Object) (bool, error) { + return AddLabel(obj, m.Value(api.PreparingDeleteLabel), strconv.FormatInt(time.Now().UnixNano(), 10)), nil } func AddLabel(po client.Object, k, v string) bool { @@ -105,7 +101,7 @@ func AddLabel(po client.Object, k, v string) bool { // IsDuringOps decides whether the Target is during ops or not // DuringOps means the Target's OpsLifecycle phase is in or after PreCheck phase and before Finish phase. -func IsDuringOps(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) bool { +func IsDuringOps(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) bool { _, hasID := checkOperatingID(m, adapter, obj) _, hasType := checkOperationType(m, adapter, obj) @@ -113,7 +109,7 @@ func IsDuringOps(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj } // Begin is used for an CRD Operator to begin a lifecycle -func Begin(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { +func Begin(m api.XSetLabelAnnotationManager, c client.Client, adapter v1alpha1.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { if obj.GetLabels() == nil { obj.SetLabels(map[string]string{}) } @@ -160,7 +156,7 @@ func Begin(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAd } // BeginWithCleaningOld is used for an CRD Operator to begin a lifecycle with cleaning the old lifecycle -func BeginWithCleaningOld(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFunc ...UpdateFunc) (updated bool, err error) { +func BeginWithCleaningOld(m api.XSetLabelAnnotationManager, c client.Client, adapter v1alpha1.LifecycleAdapter, obj client.Object, updateFunc ...UpdateFunc) (updated bool, err error) { if targetInUpdateLifecycle, err := IsLifecycleOnTarget(m, adapter.GetID(), obj); err != nil { return false, fmt.Errorf("fail to check %s TargetOpsLifecycle on Target %s/%s: %w", adapter.GetID(), obj.GetNamespace(), obj.GetName(), err) } else if targetInUpdateLifecycle { @@ -172,7 +168,7 @@ func BeginWithCleaningOld(m api.LifeCycleLabelManager, c client.Client, adapter } // AllowOps is used to check whether the TargetOpsLifecycle phase is in UPGRADE to do following operations. -func AllowOps(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { +func AllowOps(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { if !IsDuringOps(m, adapter, obj) { return nil, false } @@ -199,7 +195,7 @@ func AllowOps(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, operati } // Finish is used for an CRD Operator to finish a lifecycle -func Finish(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { +func Finish(m api.XSetLabelAnnotationManager, c client.Client, adapter v1alpha1.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { operatingID, hasID := checkOperatingID(m, adapter, obj) operationType, hasType := checkOperationType(m, adapter, obj) @@ -226,66 +222,66 @@ func Finish(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleA } // Undo is used for an CRD Operator to undo a lifecycle -func Undo(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object) error { +func Undo(m api.XSetLabelAnnotationManager, c client.Client, adapter v1alpha1.LifecycleAdapter, obj client.Object) error { setUndo(m, adapter, obj) return c.Update(context.Background(), obj) } -func checkOperatingID(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { - labelID := fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), adapter.GetID()) +func checkOperatingID(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelID := fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), adapter.GetID()) _, ok = obj.GetLabels()[labelID] return adapter.GetID(), ok } -func checkOperationType(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) (val api.OperationType, ok bool) { - labelType := fmt.Sprintf("%s/%s", m.Get(api.OperationTypeLabelPrefix), adapter.GetID()) +func checkOperationType(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) (val v1alpha1.OperationType, ok bool) { + labelType := fmt.Sprintf("%s/%s", m.Value(api.OperationTypeLabelPrefix), adapter.GetID()) labelVal := obj.GetLabels()[labelType] - val = api.OperationType(labelVal) + val = v1alpha1.OperationType(labelVal) return val, val == adapter.GetType() } -func checkOperate(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { - labelOperate := fmt.Sprintf("%s/%s", m.Get(api.OperateLabelPrefix), adapter.GetID()) +func checkOperate(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelOperate := fmt.Sprintf("%s/%s", m.Value(api.OperateLabelPrefix), adapter.GetID()) val, ok = obj.GetLabels()[labelOperate] return } -func setOperatingID(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { - labelID := fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), adapter.GetID()) +func setOperatingID(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) { + labelID := fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), adapter.GetID()) obj.GetLabels()[labelID] = fmt.Sprintf("%d", time.Now().UnixNano()) return } -func setOperationType(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { - labelType := fmt.Sprintf("%s/%s", m.Get(api.OperationTypeLabelPrefix), adapter.GetID()) +func setOperationType(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) { + labelType := fmt.Sprintf("%s/%s", m.Value(api.OperationTypeLabelPrefix), adapter.GetID()) obj.GetLabels()[labelType] = string(adapter.GetType()) return } // setOperate only for test, expected to be called by adapter -func setOperate(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { - labelOperate := fmt.Sprintf("%s/%s", m.Get(api.OperateLabelPrefix), adapter.GetID()) +func setOperate(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) { + labelOperate := fmt.Sprintf("%s/%s", m.Value(api.OperateLabelPrefix), adapter.GetID()) obj.GetLabels()[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return } -func setUndo(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { - labelUndo := fmt.Sprintf("%s/%s", m.Get(api.UndoOperationTypeLabelPrefix), adapter.GetID()) +func setUndo(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) { + labelUndo := fmt.Sprintf("%s/%s", m.Value(api.UndoOperationTypeLabelPrefix), adapter.GetID()) obj.GetLabels()[labelUndo] = string(adapter.GetType()) } -func deleteOperatingID(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { - labelID := fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), adapter.GetID()) +func deleteOperatingID(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) { + labelID := fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), adapter.GetID()) delete(obj.GetLabels(), labelID) return } -func queryByOperationType(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) sets.String { +func queryByOperationType(m api.XSetLabelAnnotationManager, adapter v1alpha1.LifecycleAdapter, obj client.Object) sets.String { res := sets.String{} valType := adapter.GetType() for k, v := range obj.GetLabels() { - if strings.HasPrefix(k, m.Get(api.OperationTypeLabelPrefix)) && v == string(valType) { + if strings.HasPrefix(k, m.Value(api.OperationTypeLabelPrefix)) && v == string(valType) { res.Insert(k) } } @@ -293,7 +289,7 @@ func queryByOperationType(m api.LifeCycleLabelManager, adapter api.LifecycleAdap return res } -func IsLifecycleOnTarget(m api.LifeCycleLabelManager, operatingID string, target client.Object) (bool, error) { +func IsLifecycleOnTarget(m api.XSetLabelAnnotationManager, operatingID string, target client.Object) (bool, error) { if target == nil { return false, fmt.Errorf("nil target") } @@ -303,13 +299,28 @@ func IsLifecycleOnTarget(m api.LifeCycleLabelManager, operatingID string, target return false, nil } - if val, ok := labels[fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), operatingID)]; ok { + if val, ok := labels[fmt.Sprintf("%s/%s", m.Value(api.OperatingLabelPrefix), operatingID)]; ok { return val != "", nil } return false, nil } +func CancelOpsLifecycle(m api.XSetLabelAnnotationManager, client client.Client, adapter v1alpha1.LifecycleAdapter, target client.Object) error { + if target == nil { + return nil + } + + // only cancel when lifecycle exist on pod + if exist, err := IsLifecycleOnTarget(m, adapter.GetID(), target); err != nil { + return fmt.Errorf("fail to check %s PodOpsLifecycle on Pod %s/%s: %w", adapter.GetID(), target.GetNamespace(), target.GetName(), err) + } else if !exist { + return nil + } + + return Undo(m, client, adapter, target) +} + func DefaultUpdateAll(target client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { for _, updateFunc := range updateFuncs { ok, updateErr := updateFunc(target) diff --git a/xset/opslifecycle/utils_test.go b/xset/opslifecycle/utils_test.go index 5616adc..0f0bb80 100644 --- a/xset/opslifecycle/utils_test.go +++ b/xset/opslifecycle/utils_test.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) const ( @@ -55,7 +56,7 @@ func TestLifecycle(t *testing.T) { a := &mockAdapter{id: "id-1", operationType: "type-1"} b := &mockAdapter{id: "id-2", operationType: "type-1"} - mgr := NewLabelManager(nil) + mgr := api.NewXSetLabelAnnotationManager() inputs := []struct { hasOperating, hasConflictID bool @@ -74,7 +75,7 @@ func TestLifecycle(t *testing.T) { { hasConflictID: true, started: false, - err: fmt.Errorf("operationType %s exists: %v", a.GetType(), sets.NewString(fmt.Sprintf("%s/%s", mgr.Get(api.OperationTypeLabelPrefix), b.GetID()))), + err: fmt.Errorf("operationType %s exists: %v", a.GetType(), sets.NewString(fmt.Sprintf("%s/%s", mgr.Value(api.OperationTypeLabelPrefix), b.GetID()))), }, { hasConflictID: true, @@ -131,14 +132,14 @@ func TestLifecycle(t *testing.T) { type mockAdapter struct { id string - operationType api.OperationType + operationType v1alpha1.OperationType } func (m *mockAdapter) GetID() string { return m.id } -func (m *mockAdapter) GetType() api.OperationType { +func (m *mockAdapter) GetType() v1alpha1.OperationType { return m.operationType } diff --git a/xset/resourcecontexts/default_adapters.go b/xset/resourcecontexts/default_adapters.go index ad852de..5a0f306 100644 --- a/xset/resourcecontexts/default_adapters.go +++ b/xset/resourcecontexts/default_adapters.go @@ -20,18 +20,20 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" - "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) -var _ api.ResourceContextAdapter = &DefaultResourceContextAdapter{} +var _ v1alpha1.ResourceContextAdapter = &DefaultResourceContextAdapter{} -var defaultResourceContextKeys = map[api.ResourceContextKeyEnum]string{ - api.EnumOwnerContextKey: "Owner", - api.EnumRevisionContextDataKey: "Revision", - api.EnumTargetDecorationRevisionKey: "TargetDecorationRevisions", - api.EnumJustCreateContextDataKey: "TargetJustCreate", - api.EnumRecreateUpdateContextDataKey: "TargetRecreateUpdate", - api.EnumScaleInContextDataKey: "ScaleIn", +var defaultResourceContextKeys = map[v1alpha1.ResourceContextKeyEnum]string{ + v1alpha1.EnumOwnerContextKey: "Owner", + v1alpha1.EnumRevisionContextDataKey: "Revision", + v1alpha1.EnumTargetDecorationRevisionKey: "TargetDecorationRevisions", + v1alpha1.EnumJustCreateContextDataKey: "TargetJustCreate", + v1alpha1.EnumRecreateUpdateContextDataKey: "TargetRecreateUpdate", + v1alpha1.EnumScaleInContextDataKey: "ScaleIn", + v1alpha1.EnumReplaceNewTargetIDContextDataKey: "ReplaceNewTargetID", + v1alpha1.EnumReplaceOriginTargetIDContextDataKey: "ReplaceOriginTargetID", } // DefaultResourceContextAdapter is the adapter to api apps.kusionstack.io.resourcecontexts @@ -41,22 +43,22 @@ func (*DefaultResourceContextAdapter) ResourceContextMeta() metav1.TypeMeta { return metav1.TypeMeta{APIVersion: appsv1alpha1.SchemeGroupVersion.String(), Kind: "ResourceContext"} } -func (*DefaultResourceContextAdapter) GetResourceContextSpec(object api.ResourceContextObject) *api.ResourceContextSpec { +func (*DefaultResourceContextAdapter) GetResourceContextSpec(object v1alpha1.ResourceContextObject) *v1alpha1.ResourceContextSpec { rc := object.(*appsv1alpha1.ResourceContext) - var contexts []api.ContextDetail + var contexts []v1alpha1.ContextDetail for i := range rc.Spec.Contexts { c := rc.Spec.Contexts[i] - contexts = append(contexts, api.ContextDetail{ + contexts = append(contexts, v1alpha1.ContextDetail{ ID: c.ID, Data: c.Data, }) } - return &api.ResourceContextSpec{ + return &v1alpha1.ResourceContextSpec{ Contexts: contexts, } } -func (*DefaultResourceContextAdapter) SetResourceContextSpec(spec *api.ResourceContextSpec, object api.ResourceContextObject) { +func (*DefaultResourceContextAdapter) SetResourceContextSpec(spec *v1alpha1.ResourceContextSpec, object v1alpha1.ResourceContextObject) { rc := object.(*appsv1alpha1.ResourceContext) var contexts []appsv1alpha1.ContextDetail for i := range spec.Contexts { @@ -69,10 +71,10 @@ func (*DefaultResourceContextAdapter) SetResourceContextSpec(spec *api.ResourceC rc.Spec.Contexts = contexts } -func (*DefaultResourceContextAdapter) GetContextKeys() map[api.ResourceContextKeyEnum]string { +func (*DefaultResourceContextAdapter) GetContextKeys() map[v1alpha1.ResourceContextKeyEnum]string { return defaultResourceContextKeys } -func (*DefaultResourceContextAdapter) NewResourceContext() api.ResourceContextObject { +func (*DefaultResourceContextAdapter) NewResourceContext() v1alpha1.ResourceContextObject { return &appsv1alpha1.ResourceContext{} } diff --git a/xset/resourcecontexts/resource_context.go b/xset/resourcecontexts/resource_context.go index daa1a04..89e679e 100644 --- a/xset/resourcecontexts/resource_context.go +++ b/xset/resourcecontexts/resource_context.go @@ -29,32 +29,32 @@ import ( clientutil "kusionstack.io/kube-utils/client" "kusionstack.io/kube-utils/controller/expectations" - "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) type ResourceContextControl interface { - AllocateID(ctx context.Context, xsetObject api.XSetObject, defaultRevision string, replicas int) (map[int]*api.ContextDetail, error) - UpdateToTargetContext(ctx context.Context, xsetObject api.XSetObject, ownedIDs map[int]*api.ContextDetail) error - ExtractAvailableContexts(diff int, ownedIDs map[int]*api.ContextDetail, targetInstanceIDSet sets.Int) []*api.ContextDetail - Get(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) (string, bool) - Contains(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) bool - Put(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) - Remove(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) + AllocateID(ctx context.Context, xsetObject v1alpha1.XSetObject, defaultRevision string, replicas int) (map[int]*v1alpha1.ContextDetail, error) + UpdateToTargetContext(ctx context.Context, xsetObject v1alpha1.XSetObject, ownedIDs map[int]*v1alpha1.ContextDetail) error + ExtractAvailableContexts(diff int, ownedIDs map[int]*v1alpha1.ContextDetail, targetInstanceIDSet sets.Int) []*v1alpha1.ContextDetail + Get(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum) (string, bool) + Contains(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum, value string) bool + Put(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum, value string) + Remove(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum) } type RealResourceContextControl struct { client.Client - xsetController api.XSetController - resourceContextAdapter api.ResourceContextAdapter - resourceContextKeys map[api.ResourceContextKeyEnum]string + xsetController v1alpha1.XSetController + resourceContextAdapter v1alpha1.ResourceContextAdapter + resourceContextKeys map[v1alpha1.ResourceContextKeyEnum]string resourceContextGVK schema.GroupVersionKind cacheExpectations expectations.CacheExpectationsInterface } func NewRealResourceContextControl( c client.Client, - xsetController api.XSetController, - resourceContextAdapter api.ResourceContextAdapter, + xsetController v1alpha1.XSetController, + resourceContextAdapter v1alpha1.ResourceContextAdapter, resourceContextGVK schema.GroupVersionKind, cacheExpectations expectations.CacheExpectationsInterface, ) ResourceContextControl { @@ -75,10 +75,10 @@ func NewRealResourceContextControl( func (r *RealResourceContextControl) AllocateID( ctx context.Context, - xsetObject api.XSetObject, + xsetObject v1alpha1.XSetObject, defaultRevision string, replicas int, -) (map[int]*api.ContextDetail, error) { +) (map[int]*v1alpha1.ContextDetail, error) { contextName := getContextName(r.xsetController, xsetObject) targetContext := r.resourceContextAdapter.NewResourceContext() notFound := false @@ -94,13 +94,13 @@ func (r *RealResourceContextControl) AllocateID( xsetSpec := r.xsetController.GetXSetSpec(xsetObject) // store all the IDs crossing Multiple workload - existingIDs := map[int]*api.ContextDetail{} + existingIDs := map[int]*v1alpha1.ContextDetail{} // only store the IDs belonging to this owner - ownedIDs := map[int]*api.ContextDetail{} + ownedIDs := map[int]*v1alpha1.ContextDetail{} resourceContextSpec := r.resourceContextAdapter.GetResourceContextSpec(targetContext) for i := range resourceContextSpec.Contexts { detail := &resourceContextSpec.Contexts[i] - if r.Contains(detail, api.EnumOwnerContextKey, xsetObject.GetName()) { + if r.Contains(detail, v1alpha1.EnumOwnerContextKey, xsetObject.GetName()) { ownedIDs[detail.ID] = detail existingIDs[detail.ID] = detail } else if xsetSpec.ScaleStrategy.Context != "" { @@ -127,13 +127,13 @@ func (r *RealResourceContextControl) AllocateID( break } - detail := &api.ContextDetail{ + detail := &v1alpha1.ContextDetail{ ID: candidateID, // TODO choose just create targets' revision according to scaleStrategy Data: map[string]string{ - r.resourceContextKeys[api.EnumOwnerContextKey]: xsetObject.GetName(), - r.resourceContextKeys[api.EnumRevisionContextDataKey]: defaultRevision, - r.resourceContextKeys[api.EnumJustCreateContextDataKey]: "true", + r.resourceContextKeys[v1alpha1.EnumOwnerContextKey]: xsetObject.GetName(), + r.resourceContextKeys[v1alpha1.EnumRevisionContextDataKey]: defaultRevision, + r.resourceContextKeys[v1alpha1.EnumJustCreateContextDataKey]: "true", }, } existingIDs[candidateID] = detail @@ -149,8 +149,8 @@ func (r *RealResourceContextControl) AllocateID( func (r *RealResourceContextControl) UpdateToTargetContext( ctx context.Context, - xSetObject api.XSetObject, - ownedIDs map[int]*api.ContextDetail, + xSetObject v1alpha1.XSetObject, + ownedIDs map[int]*v1alpha1.ContextDetail, ) error { contextName := getContextName(r.xsetController, xSetObject) targetContext := r.resourceContextAdapter.NewResourceContext() @@ -171,8 +171,8 @@ func (r *RealResourceContextControl) UpdateToTargetContext( return r.doUpdateTargetContext(ctx, xSetObject, ownedIDs, targetContext) } -func (r *RealResourceContextControl) ExtractAvailableContexts(diff int, ownedIDs map[int]*api.ContextDetail, targetInstanceIDSet sets.Int) []*api.ContextDetail { - var availableContexts []*api.ContextDetail +func (r *RealResourceContextControl) ExtractAvailableContexts(diff int, ownedIDs map[int]*v1alpha1.ContextDetail, targetInstanceIDSet sets.Int) []*v1alpha1.ContextDetail { + var availableContexts []*v1alpha1.ContextDetail if diff <= 0 { return availableContexts } @@ -193,33 +193,33 @@ func (r *RealResourceContextControl) ExtractAvailableContexts(diff int, ownedIDs return availableContexts } -func (r *RealResourceContextControl) Get(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) (string, bool) { +func (r *RealResourceContextControl) Get(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum) (string, bool) { return detail.Get(r.resourceContextKeys[enum]) } -func (r *RealResourceContextControl) Contains(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) bool { +func (r *RealResourceContextControl) Contains(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum, value string) bool { return detail.Contains(r.resourceContextKeys[enum], value) } -func (r *RealResourceContextControl) Put(detail *api.ContextDetail, enum api.ResourceContextKeyEnum, value string) { +func (r *RealResourceContextControl) Put(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum, value string) { detail.Put(r.resourceContextKeys[enum], value) } -func (r *RealResourceContextControl) Remove(detail *api.ContextDetail, enum api.ResourceContextKeyEnum) { +func (r *RealResourceContextControl) Remove(detail *v1alpha1.ContextDetail, enum v1alpha1.ResourceContextKeyEnum) { detail.Remove(r.resourceContextKeys[enum]) } func (r *RealResourceContextControl) doCreateTargetContext( ctx context.Context, - xSetObject api.XSetObject, - ownerIDs map[int]*api.ContextDetail, + xSetObject v1alpha1.XSetObject, + ownerIDs map[int]*v1alpha1.ContextDetail, ) error { contextName := getContextName(r.xsetController, xSetObject) targetContext := r.resourceContextAdapter.NewResourceContext() targetContext.SetNamespace(xSetObject.GetNamespace()) targetContext.SetName(contextName) - spec := &api.ResourceContextSpec{} + spec := &v1alpha1.ResourceContextSpec{} for i := range ownerIDs { spec.Contexts = append(spec.Contexts, *ownerIDs[i]) } @@ -233,16 +233,16 @@ func (r *RealResourceContextControl) doCreateTargetContext( func (r *RealResourceContextControl) doUpdateTargetContext( ctx context.Context, xsetObject client.Object, - ownedIDs map[int]*api.ContextDetail, - targetContext api.ResourceContextObject, + ownedIDs map[int]*v1alpha1.ContextDetail, + targetContext v1alpha1.ResourceContextObject, ) error { // store all IDs crossing all workload - existingIDs := map[int]*api.ContextDetail{} + existingIDs := map[int]*v1alpha1.ContextDetail{} // add other collaset targetContexts only if context pool enabled xsetSpec := r.xsetController.GetXSetSpec(xsetObject) resourceContextSpec := r.resourceContextAdapter.GetResourceContextSpec(targetContext) - ownerContextKey := r.resourceContextKeys[api.EnumOwnerContextKey] + ownerContextKey := r.resourceContextKeys[v1alpha1.EnumOwnerContextKey] if xsetSpec.ScaleStrategy.Context != "" { for i := range resourceContextSpec.Contexts { detail := resourceContextSpec.Contexts[i] @@ -266,7 +266,7 @@ func (r *RealResourceContextControl) doUpdateTargetContext( return r.cacheExpectations.ExpectDeletion(clientutil.ObjectKeyString(xsetObject), r.resourceContextGVK, targetContext.GetNamespace(), targetContext.GetName()) } - resourceContextSpec.Contexts = make([]api.ContextDetail, len(existingIDs)) + resourceContextSpec.Contexts = make([]v1alpha1.ContextDetail, len(existingIDs)) idx := 0 for _, contextDetail := range existingIDs { resourceContextSpec.Contexts[idx] = *contextDetail @@ -283,7 +283,7 @@ func (r *RealResourceContextControl) doUpdateTargetContext( return r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.resourceContextGVK, targetContext.GetNamespace(), targetContext.GetName(), targetContext.GetResourceVersion()) } -func getContextName(xsetControl api.XSetController, instance api.XSetObject) string { +func getContextName(xsetControl v1alpha1.XSetController, instance v1alpha1.XSetObject) string { spec := xsetControl.GetXSetSpec(instance) if spec.ScaleStrategy.Context != "" { return spec.ScaleStrategy.Context @@ -292,7 +292,7 @@ func getContextName(xsetControl api.XSetController, instance api.XSetObject) str return instance.GetName() } -type ContextDetailsByOrder []api.ContextDetail +type ContextDetailsByOrder []v1alpha1.ContextDetail func (s ContextDetailsByOrder) Len() int { return len(s) } func (s ContextDetailsByOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/xset/revisionowner/revision_owner.go b/xset/revisionowner/revision_owner.go index ba0fd17..d544326 100644 --- a/xset/revisionowner/revision_owner.go +++ b/xset/revisionowner/revision_owner.go @@ -23,22 +23,21 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" - "kusionstack.io/kube-api/apps/v1alpha1" "kusionstack.io/kube-utils/controller/history" - "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" "kusionstack.io/kube-utils/xset/xcontrol" ) var _ history.RevisionOwner = &revisionOwner{} type revisionOwner struct { - api.XSetController + v1alpha1.XSetController xcontrol.TargetControl } -func NewRevisionOwner(xsetController api.XSetController, xcontrol xcontrol.TargetControl) *revisionOwner { +func NewRevisionOwner(xsetController v1alpha1.XSetController, xcontrol xcontrol.TargetControl) *revisionOwner { return &revisionOwner{ XSetController: xsetController, TargetControl: xcontrol, @@ -51,20 +50,14 @@ func (r *revisionOwner) GetGroupVersionKind() schema.GroupVersionKind { } func (r *revisionOwner) GetMatchLabels(parent metav1.Object) map[string]string { - obj := parent.(api.XSetObject) + obj := parent.(v1alpha1.XSetObject) xsetSpec := r.XSetController.GetXSetSpec(obj) - if len(xsetSpec.Selector.MatchLabels) > 0 { - return xsetSpec.Selector.MatchLabels - } else { - return map[string]string{ - v1alpha1.ControlledByKusionStackLabelKey: "xset", - "xset.kusionstack.io/name": obj.GetName(), - } - } + // that's ok to return nil, manager will use nothingSelector instead + return xsetSpec.Selector.MatchLabels } func (r *revisionOwner) GetInUsedRevisions(parent metav1.Object) (sets.String, error) { - xSetObject, _ := parent.(api.XSetObject) + xSetObject, _ := parent.(v1alpha1.XSetObject) spec := r.XSetController.GetXSetSpec(xSetObject) status := r.XSetController.GetXSetStatus(xSetObject) @@ -89,12 +82,12 @@ func (r *revisionOwner) GetInUsedRevisions(parent metav1.Object) (sets.String, e } func (r *revisionOwner) GetCollisionCount(obj metav1.Object) *int32 { - xset := obj.(api.XSetObject) + xset := obj.(v1alpha1.XSetObject) return r.XSetController.GetXSetStatus(xset).CollisionCount } func (r *revisionOwner) GetHistoryLimit(obj metav1.Object) int32 { - xset := obj.(api.XSetObject) + xset := obj.(v1alpha1.XSetObject) return r.XSetController.GetXSetSpec(xset).HistoryLimit } @@ -103,7 +96,7 @@ func (r *revisionOwner) GetPatch(obj metav1.Object) ([]byte, error) { } func (r *revisionOwner) GetCurrentRevision(obj metav1.Object) string { - xset := obj.(api.XSetObject) + xset := obj.(v1alpha1.XSetObject) return r.XSetController.GetXSetStatus(xset).CurrentRevision } diff --git a/xset/synccontrols/const.go b/xset/synccontrols/const.go deleted file mode 100644 index a3fcf8b..0000000 --- a/xset/synccontrols/const.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package synccontrols - -// XSet labels -var ( - // TargetInstanceIDLabelKey is used to attach target instance ID on target - TargetInstanceIDLabelKey = "xset.kusionstack.io/instance-id" - // XSetUpdateIndicateLabelKey is used to indicate a target should be updated by label - XSetUpdateIndicateLabelKey = "xset.kusionstack.io/update-included" - - // TargetDeletionIndicationLabelKey indicates a target will be deleted by xset - TargetDeletionIndicationLabelKey = "xset.kusionstack.io/to-delete" - // TargetReplaceIndicationLabelKey indicates a target will be replaced by xset - TargetReplaceIndicationLabelKey = "xset.kusionstack.io/to-replace" - // TargetReplaceByReplaceUpdateLabelKey indicates a target is replaced by update by xset - TargetReplaceByReplaceUpdateLabelKey = "xset.kusionstack.io/replaced-by-replace-update" - // TargetExcludeIndicationLabelKey indicates a target will be excluded by xset - TargetExcludeIndicationLabelKey = "xset.kusionstack.io/to-exclude" - - // TargetReplacePairOriginName is used to indicate replace origin target name on the new created target - TargetReplacePairOriginName = "xset.kusionstack.io/replace-pair-origin-name" - // TargetReplacePairNewId is used to indicate the new created target instance on replace origin target - TargetReplacePairNewId = "xset.kusionstack.io/replace-pair-new-id" - - // TargetOrphanedIndicateLabelKey indicates target is orphaned - TargetOrphanedIndicateLabelKey = "xset.kusionstack.io/orphaned" -) - -// XSet annotations -var ( - // LastTargetStatusAnnotationKey is used to attach last target status on target - LastTargetStatusAnnotationKey = "xset.kusionstack.io/last-target-status" -) diff --git a/xset/synccontrols/inexclude.go b/xset/synccontrols/inexclude.go index 1f1aa4f..70b5d6e 100644 --- a/xset/synccontrols/inexclude.go +++ b/xset/synccontrols/inexclude.go @@ -20,16 +20,17 @@ import ( "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + + "kusionstack.io/kube-utils/xset/api" ) // AllowResourceExclude checks if pod or pvc is allowed to exclude -func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { +func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetLabelAnnotationManager) (bool, string) { labels := obj.GetLabels() // not controlled by ks manager if labels == nil { return false, "object's label is empty" - } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { + } else if val, exist := manager.Get(labels, api.ControlledByXSetLabel); !exist || val != "true" { return false, "object is not controlled by kusionstack system" } @@ -41,14 +42,14 @@ func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string) (bool, } // AllowResourceInclude checks if pod or pvc is allowed to include -func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { +func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string, manager api.XSetLabelAnnotationManager) (bool, string) { labels := obj.GetLabels() ownerRefs := obj.GetOwnerReferences() // not controlled by ks manager if labels == nil { return false, "object's label is empty" - } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { + } else if val, exist := manager.Get(labels, api.ControlledByXSetLabel); !exist || val != "true" { return false, "object is not controlled by kusionstack system" } diff --git a/xset/synccontrols/inexclude_test.go b/xset/synccontrols/inexclude_test.go index 7241b32..c4803ad 100644 --- a/xset/synccontrols/inexclude_test.go +++ b/xset/synccontrols/inexclude_test.go @@ -23,6 +23,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + + "kusionstack.io/kube-utils/xset/api" ) func TestAllowResourceExclude(t *testing.T) { @@ -128,7 +130,7 @@ func TestAllowResourceExclude(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceExclude(tt.obj, ownerName, ownerKind) + got, got1 := AllowResourceExclude(tt.obj, ownerName, ownerKind, api.NewXSetLabelAnnotationManager()) if got != tt.allow { t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) } @@ -288,7 +290,7 @@ func TestAllowResourceInclude(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, got1 := AllowResourceInclude(tt.obj, ownerName, ownerKind) + got, got1 := AllowResourceInclude(tt.obj, ownerName, ownerKind, api.NewXSetLabelAnnotationManager()) if got != tt.allow { t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) } diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index ce4b4f1..53d0e97 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -20,10 +20,12 @@ import ( "context" "errors" "fmt" + "strconv" "strings" "sync/atomic" "time" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,7 +34,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" clientutil "kusionstack.io/kube-utils/client" @@ -40,40 +42,40 @@ import ( "kusionstack.io/kube-utils/controller/mixin" controllerutils "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" "kusionstack.io/kube-utils/xset/opslifecycle" "kusionstack.io/kube-utils/xset/resourcecontexts" "kusionstack.io/kube-utils/xset/xcontrol" ) type SyncControl interface { - SyncTargets(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, error) + SyncTargets(ctx context.Context, instance v1alpha1.XSetObject, syncContext *SyncContext) (bool, error) - Replace(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) error + Replace(ctx context.Context, instance v1alpha1.XSetObject, syncContext *SyncContext) error - Scale(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) + Scale(ctx context.Context, instance v1alpha1.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) - Update(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) + Update(ctx context.Context, instance v1alpha1.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) - CalculateStatus(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) *api.XSetStatus + CalculateStatus(ctx context.Context, instance v1alpha1.XSetObject, syncContext *SyncContext) *v1alpha1.XSetStatus + + BatchDeleteTargetsByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error } func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, - xsetController api.XSetController, + xsetController v1alpha1.XSetController, xControl xcontrol.TargetControl, + xsetLabelAnnoManager api.XSetLabelAnnotationManager, resourceContexts resourcecontexts.ResourceContextControl, cacheExpectations expectations.CacheExpectationsInterface, ) SyncControl { - lifeCycleLabelManager := xsetController.GetLifeCycleLabelManager() - if lifeCycleLabelManager == nil { - lifeCycleLabelManager = opslifecycle.NewLabelManager(nil) - } scaleInOpsLifecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() if scaleInOpsLifecycleAdapter == nil { - scaleInOpsLifecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager} + scaleInOpsLifecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelAnnoManager: xsetLabelAnnoManager, XSetType: xsetController.XSetMeta()} } updateLifecycleAdapter := xsetController.GetUpdateOpsLifecycleAdapter() if updateLifecycleAdapter == nil { - updateLifecycleAdapter = &opslifecycle.DefaultUpdateLifecycleAdapter{LabelManager: lifeCycleLabelManager} + updateLifecycleAdapter = &opslifecycle.DefaultUpdateLifecycleAdapter{LabelAnnoManager: xsetLabelAnnoManager, XSetType: xsetController.XSetMeta()} } xMeta := xsetController.XMeta() @@ -82,23 +84,24 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, xsetGVK := xsetMeta.GroupVersionKind() updateConfig := &UpdateConfig{ - xsetController: xsetController, - client: reconcileMixIn.Client, - targetControl: xControl, - resourContextControl: resourceContexts, - recorder: reconcileMixIn.Recorder, + xsetController: xsetController, + xsetLabelAnnoMgr: xsetLabelAnnoManager, + client: reconcileMixIn.Client, + targetControl: xControl, + resourceContextControl: resourceContexts, + recorder: reconcileMixIn.Recorder, - opsLifecycleMgr: lifeCycleLabelManager, scaleInLifecycleAdapter: scaleInOpsLifecycleAdapter, updateLifecycleAdapter: updateLifecycleAdapter, cacheExpectations: cacheExpectations, targetGVK: targetGVK, } return &RealSyncControl{ - ReconcilerMixin: *reconcileMixIn, - xsetController: xsetController, - resourContextControl: resourceContexts, - xControl: xControl, + ReconcilerMixin: *reconcileMixIn, + xsetController: xsetController, + xsetLabelAnnoMgr: xsetLabelAnnoManager, + resourceContextControl: resourceContexts, + xControl: xControl, updateConfig: updateConfig, cacheExpectations: cacheExpectations, @@ -114,13 +117,14 @@ var _ SyncControl = &RealSyncControl{} type RealSyncControl struct { mixin.ReconcilerMixin - xControl xcontrol.TargetControl - xsetController api.XSetController - resourContextControl resourcecontexts.ResourceContextControl + xControl xcontrol.TargetControl + xsetController v1alpha1.XSetController + xsetLabelAnnoMgr api.XSetLabelAnnotationManager + resourceContextControl resourcecontexts.ResourceContextControl updateConfig *UpdateConfig - scaleInLifecycleAdapter api.LifecycleAdapter - updateLifecycleAdapter api.LifecycleAdapter + scaleInLifecycleAdapter v1alpha1.LifecycleAdapter + updateLifecycleAdapter v1alpha1.LifecycleAdapter cacheExpectations expectations.CacheExpectationsInterface xsetGVK schema.GroupVersionKind @@ -128,7 +132,7 @@ type RealSyncControl struct { } // SyncTargets is used to parse targetWrappers and reclaim Target instance ID -func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) ( +func (r *RealSyncControl) SyncTargets(ctx context.Context, instance v1alpha1.XSetObject, syncContext *SyncContext) ( bool, error, ) { xspec := r.xsetController.GetXSetSpec(instance) @@ -142,15 +146,17 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje return false, fmt.Errorf("fail to get filtered Targets: %w", err) } + // TODO list, adopt and retain pvcs pvcs (for pods) + toExcludeTargetNames, toIncludeTargetNames, err := r.dealIncludeExcludeTargets(ctx, instance, syncContext.FilteredTarget) if err != nil { return false, fmt.Errorf("fail to deal with include exclude targets: %s", err.Error()) } // get owned IDs - var ownedIDs map[int]*api.ContextDetail + var ownedIDs map[int]*v1alpha1.ContextDetail if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - ownedIDs, err = r.resourContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), int(RealValue(xspec.Replicas))) + ownedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), int(ptr.Deref(xspec.Replicas, 0))) syncContext.OwnedIds = ownedIDs return err }); err != nil { @@ -158,7 +164,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje } // stateless case - var targetWrappers []targetWrapper + var targetWrappers []*targetWrapper syncContext.CurrentIDs = sets.Int{} idToReclaim := sets.Int{} toDeleteTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToDelete...) @@ -166,7 +172,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje for i := range syncContext.FilteredTarget { target := syncContext.FilteredTarget[i] xName := target.GetName() - id, _ := GetInstanceID(target) + id, _ := GetInstanceID(r.xsetLabelAnnoMgr, target) toDelete := toDeleteTargetNames.Has(xName) toExclude := toExcludeTargetNames.Has(xName) @@ -175,7 +181,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje toDeleteTargetNames.Delete(xName) } if toExclude { - if targetDuringReplace(target) || toDelete { + if targetDuringReplace(r.xsetLabelAnnoMgr, target) || toDelete { // skip exclude until replace and toDelete done toExcludeTargetNames.Delete(xName) } else { @@ -186,18 +192,20 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje if target.GetDeletionTimestamp() != nil { // 1. Reclaim ID from Target which is scaling in and terminating. - if contextDetail, exist := ownedIDs[id]; exist && r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { + if contextDetail, exist := ownedIDs[id]; exist && r.resourceContextControl.Contains(contextDetail, v1alpha1.EnumScaleInContextDataKey, "true") { idToReclaim.Insert(id) } - _, replaceIndicate := target.GetLabels()[TargetReplaceIndicationLabelKey] + _, replaceIndicate := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XReplaceIndicationLabelKey) // 2. filter out Targets which are terminating and not replace indicate if !replaceIndicate { continue } } - targetWrappers = append(targetWrappers, targetWrapper{ + // TODO delete unused pvcs (for pods) + + targetWrappers = append(targetWrappers, &targetWrapper{ Object: target, ID: id, ContextDetail: ownedIDs[id], @@ -206,8 +214,8 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje ToDelete: toDelete, ToExclude: toExclude, - IsDuringScaleInOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, target), - IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, target), + IsDuringScaleInOps: opslifecycle.IsDuringOps(r.updateConfig.xsetLabelAnnoMgr, r.scaleInLifecycleAdapter, target), + IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateConfig.xsetLabelAnnoMgr, r.updateLifecycleAdapter, target), }) if id >= 0 { @@ -218,7 +226,7 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje // do include exclude targets, and skip doSync() if succeeded var inExSucceed bool if len(toExcludeTargetNames) > 0 || len(toIncludeTargetNames) > 0 { - var availableContexts []*api.ContextDetail + var availableContexts []*v1alpha1.ContextDetail var getErr error availableContexts, ownedIDs, getErr = r.getAvailableTargetIDs(ctx, len(toIncludeTargetNames), instance, syncContext) if getErr != nil { @@ -248,14 +256,11 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje syncContext.TargetWrappers = targetWrappers syncContext.OwnedIds = ownedIDs - syncContext.activeTargets = FilterOutActiveTargetWrappers(syncContext.TargetWrappers) - syncContext.replacingMap = classifyTargetReplacingMapping(syncContext.activeTargets) - return inExSucceed, nil } // dealIncludeExcludeTargets returns targets which are allowed to exclude and include -func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObject api.XSetObject, targets []client.Object) (sets.String, sets.String, error) { +func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObject v1alpha1.XSetObject, targets []client.Object) (sets.String, sets.String, error) { spec := r.xsetController.GetXSetSpec(xsetObject) ownedTargets := sets.String{} excludeTargetNames := sets.String{} @@ -263,7 +268,7 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj for _, target := range targets { ownedTargets.Insert(target.GetName()) - if _, exist := target.GetLabels()[TargetExcludeIndicationLabelKey]; exist { + if _, exist := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XExcludeIndicationLabelKey); exist { excludeTargetNames.Insert(target.GetName()) } } @@ -293,9 +298,8 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "DupExIncludedTarget", "duplicated targets %s in both excluding and including sets", strings.Join(intersection.List(), ", ")) } - // seem no need to check allow ResourceExclude, since filterTargets already filter only owned targets - toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude) - toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude) + toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude, r.xsetLabelAnnoMgr) + toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude, r.xsetLabelAnnoMgr) if notAllowedExcludeTargets.Len() > 0 { r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ExcludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to exclude, please find out the reason from target's event", notAllowedExcludeTargets.List())) } @@ -306,10 +310,10 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj } // checkAllowFunc refers to AllowResourceExclude and AllowResourceInclude -type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string) (bool, string) +type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string, labelMgr api.XSetLabelAnnotationManager) (bool, string) // allowIncludeExcludeTargets try to classify targetNames to allowedTargets and notAllowedTargets, using checkAllowFunc func -func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, targetNames []string, fn checkAllowFunc) (allowTargets, notAllowTargets sets.String, err error) { +func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset v1alpha1.XSetObject, targetNames []string, fn checkAllowFunc, labelMgr api.XSetLabelAnnotationManager) (allowTargets, notAllowTargets sets.String, err error) { allowTargets = sets.String{} notAllowTargets = sets.String{} for i := range targetNames { @@ -325,7 +329,7 @@ func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset a } // check allowance for target - if allowed, reason := fn(target, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind); !allowed { + if allowed, reason := fn(target, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind, labelMgr); !allowed { r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", fmt.Sprintf("target is not allowed to exclude/include from/to %s %s/%s: %s", r.xsetGVK.Kind, xset.GetNamespace(), xset.GetName(), reason)) notAllowTargets.Insert(targetName) @@ -337,15 +341,20 @@ func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset a } // Replace is used to replace replace-indicate targets -func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) error { +func (r *RealSyncControl) Replace(ctx context.Context, xsetObject v1alpha1.XSetObject, syncContext *SyncContext) error { var err error var needUpdateContext bool var idToReclaim sets.Int - needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(syncContext.FilteredTarget) + defer func() { + syncContext.activeTargets = FilterOutActiveTargetWrappers(syncContext.TargetWrappers) + syncContext.replacingMap = classifyTargetReplacingMapping(r.xsetLabelAnnoMgr, syncContext.activeTargets) + }() + + needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(ctx, syncContext.FilteredTarget) // delete origin targets for replace - err = BatchDeleteTargetByLabel(ctx, r.xControl, needDeleteTargets) + err = r.BatchDeleteTargetsByLabel(ctx, r.xControl, needDeleteTargets) if err != nil { r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", "delete targets by label with error: %s", err.Error()) return err @@ -360,7 +369,7 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject // create new targets for need replace targets if len(needReplaceOriginTargets) > 0 { - var availableContexts []*api.ContextDetail + var availableContexts []*v1alpha1.ContextDetail var getErr error availableContexts, syncContext.OwnedIds, getErr = r.getAvailableTargetIDs(ctx, len(needReplaceOriginTargets), xsetObject, syncContext) if getErr != nil { @@ -385,7 +394,7 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject if _, inUsed := syncContext.CurrentIDs[id]; inUsed { continue } - syncContext.TargetWrappers = append(syncContext.TargetWrappers, targetWrapper{ + syncContext.TargetWrappers = append(syncContext.TargetWrappers, &targetWrapper{ ID: id, Object: nil, ContextDetail: contextDetail, @@ -396,19 +405,19 @@ func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject return nil } -func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { +func (r *RealSyncControl) Scale(ctx context.Context, xsetObject v1alpha1.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { spec := r.xsetController.GetXSetSpec(xsetObject) - logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xsetObject)) + logger := logr.FromContext(ctx) var recordedRequeueAfter *time.Duration - diff := int(RealValue(spec.Replicas)) - len(syncContext.replacingMap) + diff := int(ptr.Deref(spec.Replicas, 0)) - len(syncContext.replacingMap) scaling := false if diff >= 0 { // trigger delete targets indicated in ScaleStrategy.TargetToDelete by label for _, targetWrapper := range syncContext.activeTargets { if targetWrapper.ToDelete { - err := BatchDeleteTargetByLabel(ctx, r.xControl, []client.Object{targetWrapper.Object}) + err := r.BatchDeleteTargetsByLabel(ctx, r.xControl, []client.Object{targetWrapper.Object}) if err != nil { return false, recordedRequeueAfter, err } @@ -424,10 +433,16 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } // find IDs and their contexts which have not been used by owned Targets - availableContext := r.resourContextControl.ExtractAvailableContexts(diff, syncContext.OwnedIds, targetInstanceIDSet) + var availableContexts []*v1alpha1.ContextDetail + var getErr error + availableContexts, syncContext.OwnedIds, getErr = r.getAvailableTargetIDs(ctx, diff, xsetObject, syncContext) + if getErr != nil { + return false, recordedRequeueAfter, getErr + } + needUpdateContext := atomic.Bool{} - succCount, err := controllerutils.SlowStartBatch(len(availableContext), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { - availableIDContext := availableContext[i] + succCount, err := controllerutils.SlowStartBatch(len(availableContexts), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { + availableIDContext := availableContexts[i] defer func() { if r.decideContextRevision(availableIDContext, syncContext.UpdatedRevision, err == nil) { needUpdateContext.Store(true) @@ -435,7 +450,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, }() // use revision recorded in Context revision := syncContext.UpdatedRevision - if revisionName, exist := r.resourContextControl.Get(availableIDContext, api.EnumRevisionContextDataKey); exist && revisionName != "" { + if revisionName, exist := r.resourceContextControl.Get(availableIDContext, v1alpha1.EnumRevisionContextDataKey); exist && revisionName != "" { for i := range syncContext.Revisions { if syncContext.Revisions[i].GetName() == revisionName { revision = syncContext.Revisions[i] @@ -445,13 +460,25 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } // scale out new Targets with updatedRevision // TODO use cache - target, err := NewTargetFrom(r.xsetController, xsetObject, revision, availableIDContext.ID) + // TODO decoration for target template + target, err := NewTargetFrom(r.xsetController, r.xsetLabelAnnoMgr, xsetObject, revision, availableIDContext.ID, + func(object client.Object) error { + if _, exist := r.resourceContextControl.Get(availableIDContext, v1alpha1.EnumJustCreateContextDataKey); exist { + r.xsetLabelAnnoMgr.Set(object.GetLabels(), api.XCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + } else { + r.xsetLabelAnnoMgr.Set(object.GetLabels(), api.XCompletingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + } + return nil + }, + r.xsetController.GetXSetTemplatePatcher(xsetObject), + ) if err != nil { return fmt.Errorf("fail to new Target from revision %s: %w", revision.GetName(), err) } + // TODO create pvcs for targets (pod) newTarget := target.DeepCopyObject().(client.Object) - logger.V(1).Info("try to create Target with revision of "+r.xsetGVK.Kind, "revision", revision.GetName()) + logger.Info("try to create Target with revision of "+r.xsetGVK.Kind, "revision", revision.GetName()) if target, err = r.xControl.CreateTarget(ctx, newTarget); err != nil { return err } @@ -459,27 +486,30 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, return r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName()) }) if needUpdateContext.Load() { - logger.V(1).Info("try to update ResourceContext for XSet after scaling out") + logger.Info("try to update ResourceContext for XSet after scaling out", "Context", syncContext.OwnedIds) if updateContextErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) + return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); updateContextErr != nil { err = errors.Join(updateContextErr, err) } } if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleOutFailed", err.Error()) + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, err, "ScaleOutFailed", err.Error()) return succCount > 0, recordedRequeueAfter, err } r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "ScaleOut", "scale out %d Target(s)", succCount) - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleOut", "") + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, nil, "ScaleOut", "") return succCount > 0, recordedRequeueAfter, err - } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleOut", "") - return false, nil, nil } - } else if diff < 0 { + } + + if diff <= 0 { + // get targets ops priority + if err := r.getTargetsOpsPriority(ctx, r.Client, syncContext.activeTargets); err != nil { + return false, recordedRequeueAfter, err + } // chose the targets to scale in - targetsToScaleIn := r.getTargetsToDelete(syncContext.activeTargets, syncContext.replacingMap, diff*-1) + targetsToScaleIn := r.getTargetsToDelete(xsetObject, syncContext.activeTargets, syncContext.replacingMap, diff*-1) // filter out Targets need to trigger TargetOpsLifecycle wrapperCh := make(chan *targetWrapper, len(targetsToScaleIn)) for i := range targetsToScaleIn { @@ -496,10 +526,10 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // trigger TargetOpsLifecycle with scaleIn OperationType logger.V(1).Info("try to begin TargetOpsLifecycle for scaling in Target in XSet", "wrapper", ObjectKeyString(object)) - // todo switch to x opslifecycle - if updated, err := opslifecycle.Begin(r.updateConfig.opsLifecycleMgr, r.Client, r.scaleInLifecycleAdapter, object); err != nil { + if updated, err := opslifecycle.Begin(r.updateConfig.xsetLabelAnnoMgr, r.Client, r.scaleInLifecycleAdapter, object); err != nil { return fmt.Errorf("fail to begin TargetOpsLifecycle for Scaling in Target %s/%s: %w", object.GetNamespace(), object.GetName(), err) } else if updated { + wrapper.IsDuringScaleInOps = true r.Recorder.Eventf(object, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin TargetOpsLifecycle for scaling in") // add an expectation for this wrapper creation, before next reconciling if err := r.cacheExpectations.ExpectUpdation(clientutil.ObjectKeyString(xsetObject), r.targetGVK, object.GetNamespace(), object.GetName(), object.GetResourceVersion()); err != nil { @@ -509,18 +539,18 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, return nil }) - scaling = succCount != 0 + scaling = succCount > 0 if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", err.Error()) + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, err, "ScaleInFailed", err.Error()) return scaling, recordedRequeueAfter, err } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, nil, "ScaleIn", "") } needUpdateContext := false for i, targetWrapper := range targetsToScaleIn { - requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, RealValue(spec.ScaleStrategy.OperationDelaySeconds), targetWrapper.Object) + requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.xsetLabelAnnoMgr, r.scaleInLifecycleAdapter, ptr.Deref(spec.ScaleStrategy.OperationDelaySeconds, 0), targetWrapper.Object) if !allowed && targetWrapper.Object.GetDeletionTimestamp() == nil { r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "Target is not allowed to scale in") continue @@ -536,9 +566,9 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, } // if Target is allowed to operate or Target has already been deleted, promte to delete Target - if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && !r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { + if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && !r.resourceContextControl.Contains(contextDetail, v1alpha1.EnumScaleInContextDataKey, "true") { needUpdateContext = true - r.resourContextControl.Put(contextDetail, api.EnumScaleInContextDataKey, "true") + r.resourceContextControl.Put(contextDetail, v1alpha1.EnumScaleInContextDataKey, "true") } if targetWrapper.GetDeletionTimestamp() != nil { @@ -550,26 +580,27 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // mark these Targets to scalingIn if needUpdateContext { - logger.V(1).Info("try to update ResourceContext for XSet when scaling in Target") + logger.Info("try to update ResourceContext for XSet when scaling in Target") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) + return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) return scaling, recordedRequeueAfter, err } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, nil, "ScaleIn", "") } } // do delete Target resource succCount, err = controllerutils.SlowStartBatch(len(wrapperCh), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { target := <-wrapperCh - logger.V(1).Info("try to scale in Target", "target", ObjectKeyString(target)) + logger.Info("try to scale in Target", "target", ObjectKeyString(target)) if err := r.xControl.DeleteTarget(ctx, target.Object); err != nil { return fmt.Errorf("fail to delete Target %s/%s when scaling in: %w", target.GetNamespace(), target.GetName(), err) } r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "TargetDeleted", "succeed to scale in Target %s/%s", target.GetNamespace(), target.GetName()) + // TODO delete pvcs if target is in update replace, or retention policy is "Deleted" return r.cacheExpectations.ExpectDeletion(clientutil.ObjectKeyString(xsetObject), r.targetGVK, target.GetNamespace(), target.GetName()) }) scaling = scaling || succCount > 0 @@ -578,46 +609,54 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "ScaleIn", "scale in %d Target(s)", succCount) } if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Target for scaling in: %s", err)) + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Target for scaling in: %s", err)) return scaling, recordedRequeueAfter, err } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, nil, "ScaleIn", "") } - - return scaling, recordedRequeueAfter, err } // reset ContextDetail.ScalingIn, if there are Targets had its TargetOpsLifecycle reverted needUpdateTargetContext := false for _, targetWrapper := range syncContext.activeTargets { - if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") && - !opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, targetWrapper) { + if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && + r.resourceContextControl.Contains(contextDetail, v1alpha1.EnumScaleInContextDataKey, "true") && !targetWrapper.IsDuringScaleInOps { needUpdateTargetContext = true - r.resourContextControl.Remove(contextDetail, api.EnumScaleInContextDataKey) + r.resourceContextControl.Remove(contextDetail, v1alpha1.EnumScaleInContextDataKey) } } if needUpdateTargetContext { logger.V(1).Info("try to update ResourceContext for XSet after scaling") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) + return r.resourceContextControl.UpdateToTargetContext(ctx, xsetObject, syncContext.OwnedIds) }); err != nil { return scaling, recordedRequeueAfter, fmt.Errorf("fail to reset ResourceContext: %w", err) } } + if !scaling { + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetScale, nil, "Scaled", "") + } + return scaling, recordedRequeueAfter, nil } -func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { - logger := r.Logger.WithValues("xset", ObjectKeyString(xsetObject)) +func (r *RealSyncControl) Update(ctx context.Context, xsetObject v1alpha1.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { + logger := logr.FromContext(ctx) var err error var recordedRequeueAfter *time.Duration + + // 0. get targets ops priority + if err := r.getTargetsOpsPriority(ctx, r.Client, syncContext.TargetWrappers); err != nil { + return false, recordedRequeueAfter, err + } + // 1. scan and analysis targets update info for active targets and PlaceHolder targets targetUpdateInfos := r.attachTargetUpdateInfo(xsetObject, syncContext) // 2. decide Target update candidates - candidates := decideTargetToUpdate(r.xsetController, xsetObject, targetUpdateInfos) + candidates := r.decideTargetToUpdate(r.xsetController, xsetObject, targetUpdateInfos) targetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) targetCh := make(chan *targetUpdateInfo, len(targetToUpdate)) updater := r.newTargetUpdater(xsetObject) @@ -625,6 +664,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, // 3. filter already updated revision, for i, targetInfo := range targetToUpdate { + // TODO check decoration and pvc template changed if targetInfo.IsUpdatedRevision { continue } @@ -641,7 +681,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, continue } - if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, targetInfo) { + if targetInfo.IsDuringScaleInOps || targetInfo.IsDuringUpdateOps { continue } @@ -658,18 +698,18 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, recordedRequeueAfter, err = updater.FilterAllowOpsTargets(ctx, candidates, syncContext.OwnedIds, syncContext, targetCh) if err != nil { AddOrUpdateCondition(syncContext.NewStatus, - api.XSetUpdate, err, "UpdateFailed", + v1alpha1.XSetUpdate, err, "UpdateFailed", fmt.Sprintf("fail to update Context for updating: %s", err)) return updating, recordedRequeueAfter, err } else { AddOrUpdateCondition(syncContext.NewStatus, - api.XSetUpdate, nil, "Updated", "") + v1alpha1.XSetUpdate, nil, "Updated", "") } // 6. update Target succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, _ error) error { targetInfo := <-targetCh - logger.V(1).Info("before target update operation", + logger.Info("before target update operation", "target", ObjectKeyString(targetInfo.Object), "revision.from", targetInfo.CurrentRevision.GetName(), "revision.to", syncContext.UpdatedRevision.GetName(), @@ -678,10 +718,9 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, ) spec := r.xsetController.GetXSetSpec(xsetObject) - isReplaceUpdate := spec.UpdateStrategy.UpdatePolicy == api.XSetReplaceTargetUpdateStrategyType - if targetInfo.IsInReplacing && !isReplaceUpdate { + if targetInfo.IsInReplace && spec.UpdateStrategy.UpdatePolicy != v1alpha1.XSetReplaceTargetUpdateStrategyType { // a replacing target should be replaced by an updated revision target when encountering upgrade - if err := updateReplaceOriginTarget(ctx, r.Client, r.Recorder, targetInfo, targetInfo.ReplacePairNewTargetInfo); err != nil { + if err := updateReplaceOriginTarget(ctx, r.Client, r.Recorder, r.xsetLabelAnnoMgr, targetInfo, targetInfo.ReplacePairNewTargetInfo); err != nil { return err } } else { @@ -695,10 +734,10 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, updating = updating || succCount > 0 if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, err, "UpdateFailed", err.Error()) + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetUpdate, err, "UpdateFailed", err.Error()) return updating, recordedRequeueAfter, err } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, nil, "Updated", "") + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetUpdate, nil, "Updated", "") } targetToUpdateSet := sets.String{} @@ -709,31 +748,44 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, succCount, err = controllerutils.SlowStartBatch(len(targetUpdateInfos), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { targetInfo := targetUpdateInfos[i] - if !targetInfo.IsDuringOps || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { + if !(targetInfo.IsDuringUpdateOps || targetInfo.IsInReplaceUpdate) || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { return nil } - // check Target is during updating, and it is finished or not - finished, msg, err := updater.GetTargetUpdateFinishStatus(ctx, targetInfo) - if err != nil { - return fmt.Errorf("failed to get target %s/%s update finished: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) + var finishByCancelUpdate bool + var updateFinished bool + var msg string + var err error + + if !targetToUpdateSet.Has(targetInfo.GetName()) { + // target is out of scope (partition or by label) and not start update yet, finish update by cancel + finishByCancelUpdate = !targetInfo.IsAllowUpdateOps + logger.Info("out of update scope", "target", ObjectKeyString(targetInfo.Object), "finishByCancelUpdate", finishByCancelUpdate) + } else if !targetInfo.IsAllowUpdateOps { + // target is in update scope, but is not start update yet, if pod is updatedRevision, just finish update by cancel + finishByCancelUpdate = targetInfo.IsUpdatedRevision + } else { + // target is in update scope and allowed to update, check and finish update gracefully + if updateFinished, msg, err = updater.GetTargetUpdateFinishStatus(ctx, targetInfo); err != nil { + return fmt.Errorf("failed to get target %s/%s update finished: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) + } else if !updateFinished { + r.Recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "WaitingUpdateReady", + "waiting for target %s/%s to update finished: %s", + targetInfo.GetNamespace(), targetInfo.GetName(), msg) + } } - if finished { - if err := updater.FinishUpdateTarget(ctx, targetInfo); err != nil { - return err + if updateFinished || finishByCancelUpdate { + if err := updater.FinishUpdateTarget(ctx, targetInfo, finishByCancelUpdate); err != nil { + return fmt.Errorf("failed to finish target %s/%s update: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) } r.Recorder.Eventf(targetInfo.Object, corev1.EventTypeNormal, "UpdateTargetFinished", "target %s/%s is finished for upgrade to revision %s", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.UpdateRevision.GetName()) - } else { - r.Recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "WaitingUpdateReady", - "waiting for target %s/%s to update finished: %s", - targetInfo.GetNamespace(), targetInfo.GetName(), msg) } return nil @@ -742,7 +794,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, return updating || succCount > 0, recordedRequeueAfter, err } -func (r *RealSyncControl) CalculateStatus(_ context.Context, instance api.XSetObject, syncContext *SyncContext) *api.XSetStatus { +func (r *RealSyncControl) CalculateStatus(_ context.Context, instance v1alpha1.XSetObject, syncContext *SyncContext) *v1alpha1.XSetStatus { newStatus := syncContext.NewStatus newStatus.ObservedGeneration = instance.GetGeneration() @@ -761,8 +813,8 @@ func (r *RealSyncControl) CalculateStatus(_ context.Context, instance api.XSetOb updatedReplicas++ } - if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, targetWrapper) || - opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, targetWrapper) { + if opslifecycle.IsDuringOps(r.updateConfig.xsetLabelAnnoMgr, r.scaleInLifecycleAdapter, targetWrapper) || + opslifecycle.IsDuringOps(r.updateConfig.xsetLabelAnnoMgr, r.updateLifecycleAdapter, targetWrapper) { operatingReplicas++ } @@ -807,38 +859,38 @@ func (r *RealSyncControl) CalculateStatus(_ context.Context, instance api.XSetOb func (r *RealSyncControl) getAvailableTargetIDs( ctx context.Context, want int, - instance api.XSetObject, + instance v1alpha1.XSetObject, syncContext *SyncContext, -) ([]*api.ContextDetail, map[int]*api.ContextDetail, error) { +) ([]*v1alpha1.ContextDetail, map[int]*v1alpha1.ContextDetail, error) { ownedIDs := syncContext.OwnedIds currentIDs := syncContext.CurrentIDs - availableContexts := r.resourContextControl.ExtractAvailableContexts(want, ownedIDs, currentIDs) + availableContexts := r.resourceContextControl.ExtractAvailableContexts(want, ownedIDs, currentIDs) if len(availableContexts) >= want { return availableContexts, ownedIDs, nil } diff := want - len(availableContexts) - var newOwnedIDs map[int]*api.ContextDetail + var newOwnedIDs map[int]*v1alpha1.ContextDetail var err error if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - newOwnedIDs, err = r.resourContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), len(ownedIDs)+diff) + newOwnedIDs, err = r.resourceContextControl.AllocateID(ctx, instance, syncContext.UpdatedRevision.GetName(), len(ownedIDs)+diff) return err }); err != nil { return nil, ownedIDs, fmt.Errorf("fail to allocate IDs using context when include Targets: %w", err) } - return r.resourContextControl.ExtractAvailableContexts(want, newOwnedIDs, currentIDs), newOwnedIDs, nil + return r.resourceContextControl.ExtractAvailableContexts(want, newOwnedIDs, currentIDs), newOwnedIDs, nil } // reclaimOwnedIDs delete and reclaim unused IDs func (r *RealSyncControl) reclaimOwnedIDs( ctx context.Context, needUpdateContext bool, - xset api.XSetObject, + xset v1alpha1.XSetObject, idToReclaim sets.Int, - ownedIDs map[int]*api.ContextDetail, + ownedIDs map[int]*v1alpha1.ContextDetail, currentIDs sets.Int, ) error { // TODO stateful case @@ -848,7 +900,7 @@ func (r *RealSyncControl) reclaimOwnedIDs( if _, exist := currentIDs[id]; exist { continue } - if r.resourContextControl.Contains(contextDetail, api.EnumScaleInContextDataKey, "true") { + if r.resourceContextControl.Contains(contextDetail, v1alpha1.EnumScaleInContextDataKey, "true") { idToReclaim.Insert(id) } } @@ -866,7 +918,7 @@ func (r *RealSyncControl) reclaimOwnedIDs( logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xset)) logger.V(1).Info("try to update ResourceContext for XSet when sync") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return r.resourContextControl.UpdateToTargetContext(ctx, xset, ownedIDs) + return r.resourceContextControl.UpdateToTargetContext(ctx, xset, ownedIDs) }); err != nil { return fmt.Errorf("fail to update ResourceContextControl when reclaiming IDs: %w", err) } @@ -874,35 +926,51 @@ func (r *RealSyncControl) reclaimOwnedIDs( return nil } +// getTargetsOpsPriority try to set targets' ops priority +func (r *RealSyncControl) getTargetsOpsPriority(ctx context.Context, c client.Client, targets []*targetWrapper) error { + _, err := controllerutils.SlowStartBatch(len(targets), controllerutils.SlowStartInitialBatchSize, true, func(i int, _ error) error { + if targets[i].PlaceHolder || targets[i].Object == nil || targets[i].OpsPriority != nil { + return nil + } + var iErr error + targets[i].OpsPriority, iErr = r.xsetController.GetXOpsPriority(ctx, c, targets[i].Object) + if iErr != nil { + return fmt.Errorf("failed to get target %s/%s ops priority: %w", targets[i].Object.GetNamespace(), targets[i].Object.GetName(), iErr) + } + return nil + }) + return err +} + // FilterOutActiveTargetWrappers filter out non placeholder targets -func FilterOutActiveTargetWrappers(targets []targetWrapper) []*targetWrapper { +func FilterOutActiveTargetWrappers(targets []*targetWrapper) []*targetWrapper { var filteredTargetWrappers []*targetWrapper for i, target := range targets { if target.PlaceHolder { continue } - filteredTargetWrappers = append(filteredTargetWrappers, &targets[i]) + filteredTargetWrappers = append(filteredTargetWrappers, targets[i]) } return filteredTargetWrappers } -func targetDuringReplace(target client.Object) bool { +func targetDuringReplace(labelMgr api.XSetLabelAnnotationManager, target client.Object) bool { labels := target.GetLabels() if labels == nil { return false } - _, replaceIndicate := labels[TargetReplaceIndicationLabelKey] - _, replaceOriginTarget := labels[TargetReplacePairNewId] - _, replaceNewTarget := labels[TargetReplacePairOriginName] + _, replaceIndicate := labelMgr.Get(labels, api.XReplaceIndicationLabelKey) + _, replaceOriginTarget := labelMgr.Get(labels, api.XReplacePairOriginName) + _, replaceNewTarget := labelMgr.Get(labels, api.XReplacePairNewId) return replaceIndicate || replaceOriginTarget || replaceNewTarget } -// BatchDeleteTargetByLabel try to trigger target deletion by to-delete label -func BatchDeleteTargetByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error { +// BatchDeleteTargetsByLabel try to trigger target deletion by to-delete label +func (r *RealSyncControl) BatchDeleteTargetsByLabel(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error { _, err := controllerutils.SlowStartBatch(len(needDeleteTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { target := needDeleteTargets[i] - if _, exist := target.GetLabels()[appsv1alpha1.PodDeletionIndicationLabelKey]; !exist { - patch := client.RawPatch(types.StrategicMergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%q":"%d"}}}`, appsv1alpha1.PodDeletionIndicationLabelKey, time.Now().UnixNano()))) + if _, exist := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XDeletionIndicationLabelKey); !exist { + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%d"}}}`, r.xsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) // nolint if err := targetControl.PatchTarget(ctx, target, patch); err != nil { return fmt.Errorf("failed to delete target when syncTargets %s/%s/%w", target.GetNamespace(), target.GetName(), err) } @@ -913,24 +981,24 @@ func BatchDeleteTargetByLabel(ctx context.Context, targetControl xcontrol.Target } // decideContextRevision decides revision for 3 target create types: (1) just create, (2) upgrade by recreate, (3) delete and recreate -func (r *RealSyncControl) decideContextRevision(contextDetail *api.ContextDetail, updatedRevision *appsv1.ControllerRevision, createSucceeded bool) bool { +func (r *RealSyncControl) decideContextRevision(contextDetail *v1alpha1.ContextDetail, updatedRevision *appsv1.ControllerRevision, createSucceeded bool) bool { needUpdateContext := false if !createSucceeded { - if r.resourContextControl.Contains(contextDetail, api.EnumJustCreateContextDataKey, "true") { + if r.resourceContextControl.Contains(contextDetail, v1alpha1.EnumJustCreateContextDataKey, "true") { // TODO choose just create targets' revision according to scaleStrategy - r.resourContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) - r.resourContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) + r.resourceContextControl.Put(contextDetail, v1alpha1.EnumRevisionContextDataKey, updatedRevision.GetName()) + r.resourceContextControl.Remove(contextDetail, v1alpha1.EnumTargetDecorationRevisionKey) needUpdateContext = true - } else if r.resourContextControl.Contains(contextDetail, api.EnumRecreateUpdateContextDataKey, "true") { - r.resourContextControl.Put(contextDetail, api.EnumRevisionContextDataKey, updatedRevision.GetName()) - r.resourContextControl.Remove(contextDetail, api.EnumTargetDecorationRevisionKey) + } else if r.resourceContextControl.Contains(contextDetail, v1alpha1.EnumRecreateUpdateContextDataKey, "true") { + r.resourceContextControl.Put(contextDetail, v1alpha1.EnumRevisionContextDataKey, updatedRevision.GetName()) + r.resourceContextControl.Remove(contextDetail, v1alpha1.EnumTargetDecorationRevisionKey) needUpdateContext = true } // if target is delete and recreate, never change revisionKey } else { // TODO delete ID if create succeeded - r.resourContextControl.Remove(contextDetail, api.EnumJustCreateContextDataKey) - r.resourContextControl.Remove(contextDetail, api.EnumRecreateUpdateContextDataKey) + r.resourceContextControl.Remove(contextDetail, v1alpha1.EnumJustCreateContextDataKey) + r.resourceContextControl.Remove(contextDetail, v1alpha1.EnumRecreateUpdateContextDataKey) needUpdateContext = true } return needUpdateContext diff --git a/xset/synccontrols/types.go b/xset/synccontrols/types.go index db07c85..204652c 100644 --- a/xset/synccontrols/types.go +++ b/xset/synccontrols/types.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) type SyncContext struct { @@ -33,21 +33,21 @@ type SyncContext struct { ExistingSubResource []client.Object FilteredTarget []client.Object - TargetWrappers []targetWrapper + TargetWrappers []*targetWrapper activeTargets []*targetWrapper replacingMap map[string]*targetWrapper CurrentIDs sets.Int - OwnedIds map[int]*api.ContextDetail + OwnedIds map[int]*v1alpha1.ContextDetail - NewStatus *api.XSetStatus + NewStatus *v1alpha1.XSetStatus } type targetWrapper struct { // parameters must be set during creation client.Object ID int - ContextDetail *api.ContextDetail + ContextDetail *v1alpha1.ContextDetail PlaceHolder bool ToDelete bool @@ -55,6 +55,8 @@ type targetWrapper struct { IsDuringScaleInOps bool IsDuringUpdateOps bool + + OpsPriority *v1alpha1.OpsPriority } type targetUpdateInfo struct { @@ -72,16 +74,17 @@ type targetUpdateInfo struct { // carry the desired update revision UpdateRevision *appsv1.ControllerRevision - // indicates the TargetOpsLifecycle is started. - IsDuringOps bool + // TODO decoration revisions + // indicates operate is allowed for TargetOpsLifecycle. - IsAllowOps bool + IsAllowUpdateOps bool // requeue after for operationDelaySeconds RequeueForOperationDelay *time.Duration // for replace update // judge target in replace updating - IsInReplacing bool + IsInReplace bool + IsInReplaceUpdate bool // replace new created target ReplacePairNewTargetInfo *targetUpdateInfo diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index b8c1648..a9b9ad8 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -34,20 +35,28 @@ import ( clientutil "kusionstack.io/kube-utils/client" controllerutils "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" "kusionstack.io/kube-utils/xset/opslifecycle" ) -const ( - ReplaceNewTargetIDContextDataKey = "ReplaceNewTargetID" - ReplaceOriginTargetIDContextDataKey = "ReplaceOriginTargetID" -) - -func (r *RealSyncControl) cleanReplaceTargetLabels(ctx context.Context, needCleanLabelTargets []client.Object, targetsNeedCleanLabels [][]string, ownedIDs map[int]*api.ContextDetail, currentIDs sets.Int) (bool, sets.Int, error) { +func (r *RealSyncControl) cleanReplaceTargetLabels( + ctx context.Context, + needCleanLabelTargets []client.Object, + targetsNeedCleanLabels [][]string, + ownedIDs map[int]*v1alpha1.ContextDetail, + currentIDs sets.Int, +) (bool, sets.Int, error) { + logger := logr.FromContext(ctx) needUpdateContext := false needDeleteTargetsIDs := sets.Int{} - mapOriginToNewTargetContext := mapReplaceOriginToNewTargetContext(ownedIDs) - mapNewToOriginTargetContext := mapReplaceNewToOriginTargetContext(ownedIDs) - _, err := controllerutils.SlowStartBatch(len(needCleanLabelTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + mapOriginToNewTargetContext := r.mapReplaceOriginToNewTargetContext(ownedIDs) + mapNewToOriginTargetContext := r.mapReplaceNewToOriginTargetContext(ownedIDs) + _, err := controllerutils.SlowStartBatch(len(needCleanLabelTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { + defer func() { + if err == nil { + logger.Info("cleanReplaceTargetLabels clean replace labels success", "kind", needCleanLabelTargets[i].GetObjectKind(), "target", needCleanLabelTargets[i].GetName(), "labels", targetsNeedCleanLabels[i]) + } + }() target := needCleanLabelTargets[i] needCleanLabels := targetsNeedCleanLabels[i] var deletePatch []map[string]string @@ -58,32 +67,32 @@ func (r *RealSyncControl) cleanReplaceTargetLabels(ctx context.Context, needClea } deletePatch = append(deletePatch, patchOperation) // replace finished, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete origin Target's ID - if labelKey == TargetReplacePairOriginName { + if labelKey == r.xsetLabelAnnoMgr.Value(api.XReplacePairOriginName) { needUpdateContext = true - newTargetId, _ := GetInstanceID(target) + newTargetId, _ := GetInstanceID(r.xsetLabelAnnoMgr, target) if originTargetContext, exist := mapOriginToNewTargetContext[newTargetId]; exist && originTargetContext != nil { - originTargetContext.Remove(ReplaceNewTargetIDContextDataKey) + r.resourceContextControl.Remove(originTargetContext, v1alpha1.EnumReplaceNewTargetIDContextDataKey) if _, exist := currentIDs[originTargetContext.ID]; !exist { needDeleteTargetsIDs.Insert(originTargetContext.ID) } } if contextDetail, exist := ownedIDs[newTargetId]; exist { - contextDetail.Remove(ReplaceOriginTargetIDContextDataKey) + r.resourceContextControl.Remove(contextDetail, v1alpha1.EnumReplaceOriginTargetIDContextDataKey) } } // replace canceled, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete new Target's ID - _, replaceIndicate := target.GetLabels()[TargetReplaceIndicationLabelKey] - if !replaceIndicate && labelKey == TargetReplacePairNewId { + _, replaceIndicate := r.xsetLabelAnnoMgr.Get(target.GetLabels(), api.XReplaceIndicationLabelKey) + if !replaceIndicate && labelKey == r.xsetLabelAnnoMgr.Value(api.XReplacePairNewId) { needUpdateContext = true - originTargetId, _ := GetInstanceID(target) + originTargetId, _ := GetInstanceID(r.xsetLabelAnnoMgr, target) if newTargetContext, exist := mapNewToOriginTargetContext[originTargetId]; exist && newTargetContext != nil { - newTargetContext.Remove(ReplaceOriginTargetIDContextDataKey) + r.resourceContextControl.Remove(newTargetContext, v1alpha1.EnumReplaceOriginTargetIDContextDataKey) if _, exist := currentIDs[newTargetContext.ID]; !exist { needDeleteTargetsIDs.Insert(newTargetContext.ID) } } if contextDetail, exist := ownedIDs[originTargetId]; exist { - contextDetail.Remove(ReplaceNewTargetIDContextDataKey) + r.resourceContextControl.Remove(contextDetail, v1alpha1.EnumReplaceNewTargetIDContextDataKey) } } } @@ -103,46 +112,58 @@ func (r *RealSyncControl) cleanReplaceTargetLabels(ctx context.Context, needClea func (r *RealSyncControl) replaceOriginTargets( ctx context.Context, - instance api.XSetObject, + instance v1alpha1.XSetObject, syncContext *SyncContext, needReplaceOriginTargets []client.Object, - ownedIDs map[int]*api.ContextDetail, - availableContexts []*api.ContextDetail, + ownedIDs map[int]*v1alpha1.ContextDetail, + availableContexts []*v1alpha1.ContextDetail, ) (int, error) { - mapNewToOriginTargetContext := mapReplaceNewToOriginTargetContext(ownedIDs) + logger := logr.FromContext(ctx) + mapNewToOriginTargetContext := r.mapReplaceNewToOriginTargetContext(ownedIDs) successCount, err := controllerutils.SlowStartBatch(len(needReplaceOriginTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { originTarget := needReplaceOriginTargets[i] - originTargetId, _ := GetInstanceID(originTarget) + originTargetId, _ := GetInstanceID(r.xsetLabelAnnoMgr, originTarget) + + if ownedIDs[originTargetId] == nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "OriginTargetContext", "cannot found resource context id %d of origin target %s/%s", originTargetId, originTarget.GetNamespace(), originTarget.GetName()) + return fmt.Errorf("cannot found context for replace origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) + } - replaceRevision := getReplaceRevision(originTarget, syncContext) + replaceRevision := r.getReplaceRevision(originTarget, syncContext) // create target using update revision if replaced by update, otherwise using current revision - newTarget, err := NewTargetFrom(r.xsetController, instance, replaceRevision, originTargetId) + newTarget, err := NewTargetFrom(r.xsetController, r.xsetLabelAnnoMgr, instance, replaceRevision, originTargetId, + r.xsetController.GetXSetTemplatePatcher(instance)) if err != nil { return err } // add instance id and replace pair label - var instanceId string - var newTargetContext *api.ContextDetail + var newInstanceId string + var newTargetContext *v1alpha1.ContextDetail if contextDetail, exist := mapNewToOriginTargetContext[originTargetId]; exist && contextDetail != nil { newTargetContext = contextDetail // reuse targetContext ID if pair-relation exists - instanceId = fmt.Sprintf("%d", newTargetContext.ID) - newTarget.GetLabels()[TargetInstanceIDLabelKey] = instanceId + newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) + r.xsetLabelAnnoMgr.Set(newTarget.GetLabels(), api.XInstanceIdLabelKey, newInstanceId) + logger.Info("replaceOriginTargets", "try to reuse new pod resourceContext id", newInstanceId) } else { if availableContexts[i] == nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "AvailableContext", "cannot found available context for replace origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) return fmt.Errorf("cannot found available context for replace new target when replacing origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) } newTargetContext = availableContexts[i] // add replace pair-relation to targetContexts for originTarget and newTarget - instanceId = fmt.Sprintf("%d", newTargetContext.ID) - newTarget.GetLabels()[TargetInstanceIDLabelKey] = instanceId - ownedIDs[originTargetId].Put(ReplaceNewTargetIDContextDataKey, instanceId) - ownedIDs[newTargetContext.ID].Put(ReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) - r.resourContextControl.Remove(ownedIDs[newTargetContext.ID], api.EnumJustCreateContextDataKey) + newInstanceId = fmt.Sprintf("%d", newTargetContext.ID) + r.xsetLabelAnnoMgr.Set(newTarget.GetLabels(), api.XInstanceIdLabelKey, newInstanceId) + r.resourceContextControl.Put(ownedIDs[originTargetId], v1alpha1.EnumReplaceNewTargetIDContextDataKey, newInstanceId) + r.resourceContextControl.Put(ownedIDs[newTargetContext.ID], v1alpha1.EnumReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) + r.resourceContextControl.Remove(ownedIDs[newTargetContext.ID], v1alpha1.EnumJustCreateContextDataKey) } - newTarget.GetLabels()[TargetReplacePairOriginName] = originTarget.GetName() - r.resourContextControl.Put(newTargetContext, api.EnumRevisionContextDataKey, replaceRevision.GetName()) + r.xsetLabelAnnoMgr.Set(newTarget.GetLabels(), api.XReplacePairOriginName, originTarget.GetName()) + r.xsetLabelAnnoMgr.Set(newTarget.GetLabels(), api.XCreatingLabel, strconv.FormatInt(time.Now().UnixNano(), 10)) + r.resourceContextControl.Put(newTargetContext, v1alpha1.EnumRevisionContextDataKey, replaceRevision.GetName()) + + // TODO create pvcs for new target (pod) if newCreatedTarget, err := r.xControl.CreateTarget(ctx, newTarget); err == nil { r.Recorder.Eventf(originTarget, @@ -153,11 +174,16 @@ func (r *RealSyncControl) replaceOriginTargets( originTarget.GetName(), replaceRevision.GetName()) - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, TargetReplacePairNewId, instanceId))) + if err := r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(instance), r.targetGVK, newTarget.GetNamespace(), newTarget.GetName()); err != nil { + return err + } + + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, r.xsetLabelAnnoMgr.Value(api.XReplacePairNewId), newInstanceId))) if err = r.xControl.PatchTarget(ctx, originTarget, patch); err != nil { return fmt.Errorf("fail to update origin target %s/%s pair label %s when updating by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), newCreatedTarget.GetName(), err.Error()) } - return r.cacheExpectations.ExpectCreation(clientutil.ObjectKeyString(instance), r.targetGVK, newTarget.GetNamespace(), newTarget.GetName()) + logger.Info("replaceOriginTargets", "replacing originTarget", originTarget.GetName(), "originTargetId", originTargetId, "newTargetContextID", newInstanceId) + return nil } else { r.Recorder.Eventf(originTarget, corev1.EventTypeNormal, @@ -174,14 +200,17 @@ func (r *RealSyncControl) replaceOriginTargets( return successCount, err } -func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needReplaceTargets, needCleanLabelTargets []client.Object, targetNeedCleanLabels [][]string, needDeleteTargets []client.Object) { +func (r *RealSyncControl) dealReplaceTargets(ctx context.Context, targets []client.Object) ( + needReplaceTargets, needCleanLabelTargets []client.Object, targetNeedCleanLabels [][]string, needDeleteTargets []client.Object, +) { + logger := logr.FromContext(ctx) targetInstanceIdMap := make(map[string]client.Object) targetNameMap := make(map[string]client.Object) for _, target := range targets { targetLabels := target.GetLabels() - if instanceId, ok := targetLabels[TargetInstanceIDLabelKey]; ok { + if instanceId, ok := r.xsetLabelAnnoMgr.Get(targetLabels, api.XInstanceIdLabelKey); ok { targetInstanceIdMap[instanceId] = target } targetNameMap[target.GetName()] = target @@ -192,24 +221,25 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needRepla targetLabels := target.GetLabels() // no replace indication label - if _, exist := targetLabels[TargetReplaceIndicationLabelKey]; !exist { + if _, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplaceIndicationLabelKey); !exist { continue } // origin target is about to scaleIn, skip replace - if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, target) { + if opslifecycle.IsDuringOps(r.updateConfig.xsetLabelAnnoMgr, r.scaleInLifecycleAdapter, target) { + logger.Info("dealReplaceTargets", "target is during scaleIn ops lifecycle, skip replacing", target.GetName()) continue } // target is replace new created target, skip replace - if originTargetName, exist := targetLabels[TargetReplacePairOriginName]; exist { + if originTargetName, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairOriginName); exist { if _, exist := targetNameMap[originTargetName]; exist { continue } } // target already has a new created target for replacement - if newPairTargetId, exist := targetLabels[TargetReplacePairNewId]; exist { + if newPairTargetId, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairNewId); exist { if _, exist := targetInstanceIdMap[newPairTargetId]; exist { continue } @@ -220,30 +250,30 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needRepla for _, target := range targets { targetLabels := target.GetLabels() - _, replaceByUpdate := targetLabels[TargetReplaceByReplaceUpdateLabelKey] + _, replaceByUpdate := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplaceByReplaceUpdateLabelKey) var needCleanLabels []string // target is replace new created target, skip replace - if originTargetName, exist := targetLabels[TargetReplacePairOriginName]; exist { + if originTargetName, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairOriginName); exist { // replace pair origin target is not exist, clean label. if originTarget, exist := targetNameMap[originTargetName]; !exist { - needCleanLabels = append(needCleanLabels, TargetReplacePairOriginName) - } else if originTarget.GetLabels()[TargetReplaceIndicationLabelKey] == "" { + needCleanLabels = append(needCleanLabels, r.xsetLabelAnnoMgr.Value(api.XReplacePairOriginName)) + } else if _, exist := r.xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceIndicationLabelKey); !exist { // replace canceled, delete replace new target if new target is not service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); !serviceAvailable { + if !r.xsetController.CheckAvailable(target) { needDeleteTargets = append(needDeleteTargets, target) } } else if !replaceByUpdate { // not replace update, delete origin target when new created target is service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); serviceAvailable { + if r.xsetController.CheckAvailable(target) { needDeleteTargets = append(needDeleteTargets, originTarget) } } } - if newPairTargetId, exist := targetLabels[TargetReplacePairNewId]; exist { + if newPairTargetId, exist := r.xsetLabelAnnoMgr.Get(targetLabels, api.XReplacePairNewId); exist { if _, exist := targetInstanceIdMap[newPairTargetId]; !exist { - needCleanLabels = append(needCleanLabels, TargetReplacePairNewId) + needCleanLabels = append(needCleanLabels, r.xsetLabelAnnoMgr.Value(api.XReplacePairNewId)) } } @@ -259,6 +289,7 @@ func updateReplaceOriginTarget( ctx context.Context, c client.Client, recorder record.EventRecorder, + xsetLabelAnnoMgr api.XSetLabelAnnotationManager, originTargetUpdateInfo, newTargetUpdateInfo *targetUpdateInfo, ) error { originTarget := originTargetUpdateInfo.Object @@ -266,10 +297,10 @@ func updateReplaceOriginTarget( // 1. delete the new target if not updated if newTargetUpdateInfo != nil { newTarget := newTargetUpdateInfo.Object - _, deletionIndicate := newTarget.GetLabels()[TargetDeletionIndicationLabelKey] + _, deletionIndicate := xsetLabelAnnoMgr.Get(newTarget.GetLabels(), api.XDeletionIndicationLabelKey) currentRevision, exist := newTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] if exist && currentRevision != originTargetUpdateInfo.UpdateRevision.GetName() && !deletionIndicate { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, xsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) if patchErr := c.Patch(ctx, newTarget, patch); patchErr != nil { err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", newTarget.GetNamespace(), newTarget.GetName(), patchErr.Error()) @@ -286,11 +317,11 @@ func updateReplaceOriginTarget( } // 2. replace the origin target with updated target - _, replaceIndicate := originTarget.GetLabels()[TargetReplaceIndicationLabelKey] - _, replaceByUpdate := originTarget.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] - if !replaceIndicate || !replaceByUpdate { + _, replaceIndicate := xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceIndicationLabelKey) + replaceRevision, replaceByUpdate := xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceByReplaceUpdateLabelKey) + if !replaceIndicate || !replaceByUpdate || replaceRevision != originTargetUpdateInfo.UpdateRevision.Name { now := time.Now().UnixNano() - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, TargetReplaceIndicationLabelKey, now, TargetReplaceByReplaceUpdateLabelKey, originTargetUpdateInfo.UpdateRevision.GetName()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, xsetLabelAnnoMgr.Value(api.XReplaceIndicationLabelKey), now, xsetLabelAnnoMgr.Value(api.XReplaceByReplaceUpdateLabelKey), originTargetUpdateInfo.UpdateRevision.Name))) if err := c.Patch(ctx, originTarget, patch); err != nil { return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), err.Error()) } @@ -307,11 +338,11 @@ func updateReplaceOriginTarget( } // getReplaceRevision finds replaceNewTarget's revision from originTarget -func getReplaceRevision(originTarget client.Object, syncContext *SyncContext) *appsv1.ControllerRevision { +func (r *RealSyncControl) getReplaceRevision(originTarget client.Object, syncContext *SyncContext) *appsv1.ControllerRevision { // replace update, first find revision from label, if revision not found, just replace with updated revision - if updateRevisionName, exist := originTarget.GetLabels()[TargetReplaceByReplaceUpdateLabelKey]; exist { + if updateRevisionName, exist := r.xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplaceByReplaceUpdateLabelKey); exist { for _, rv := range syncContext.Revisions { - if updateRevisionName == rv.GetName() { + if updateRevisionName == rv.Name { return rv } } @@ -334,7 +365,7 @@ func getReplaceRevision(originTarget client.Object, syncContext *SyncContext) *a } // classify the pair relationship for Target replacement. -func classifyTargetReplacingMapping(targetWrappers []*targetWrapper) map[string]*targetWrapper { +func classifyTargetReplacingMapping(xsetLabelAnnoMgr api.XSetLabelAnnotationManager, targetWrappers []*targetWrapper) map[string]*targetWrapper { targetNameMap := make(map[string]*targetWrapper) targetIdMap := make(map[string]*targetWrapper) for _, targetWrapper := range targetWrappers { @@ -349,16 +380,17 @@ func classifyTargetReplacingMapping(targetWrappers []*targetWrapper) map[string] continue } name := targetWrapper.GetName() - if replacePairNewIdStr, exist := targetWrapper.GetLabels()[TargetReplacePairNewId]; exist { + if replacePairNewIdStr, exist := xsetLabelAnnoMgr.Get(targetWrapper.GetLabels(), api.XReplacePairNewId); exist { if pairNewTarget, exist := targetIdMap[replacePairNewIdStr]; exist { replaceTargetMapping[name] = pairNewTarget // if one of pair targets is to Exclude, both targets should not scaleIn targetWrapper.ToExclude = targetWrapper.ToExclude || pairNewTarget.ToExclude continue } - } else if replaceOriginStr, exist := targetWrapper.GetLabels()[TargetReplacePairOriginName]; exist { + } else if replaceOriginStr, exist := xsetLabelAnnoMgr.Get(targetWrapper.GetLabels(), api.XReplacePairOriginName); exist { if originTarget, exist := targetNameMap[replaceOriginStr]; exist { - if originTarget.GetLabels()[TargetReplacePairNewId] == strconv.Itoa(targetWrapper.ID) { + id, exist := xsetLabelAnnoMgr.Get(originTarget.GetLabels(), api.XReplacePairNewId) + if exist && id == strconv.Itoa(targetWrapper.ID) { continue } } @@ -370,13 +402,14 @@ func classifyTargetReplacingMapping(targetWrappers []*targetWrapper) map[string] return replaceTargetMapping } -func mapReplaceNewToOriginTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { - mapNewToOriginTargetContext := make(map[int]*api.ContextDetail) +func (r *RealSyncControl) mapReplaceNewToOriginTargetContext(ownedIDs map[int]*v1alpha1.ContextDetail) map[int]*v1alpha1.ContextDetail { + mapNewToOriginTargetContext := make(map[int]*v1alpha1.ContextDetail) for id, contextDetail := range ownedIDs { - if val, exist := contextDetail.Data[ReplaceNewTargetIDContextDataKey]; exist { + if val, exist := r.resourceContextControl.Get(contextDetail, v1alpha1.EnumReplaceNewTargetIDContextDataKey); exist { newTargetId, _ := strconv.ParseInt(val, 10, 32) newTargetContextDetail, exist := ownedIDs[int(newTargetId)] - if exist && newTargetContextDetail.Data[ReplaceOriginTargetIDContextDataKey] == strconv.Itoa(id) { + originTargetId, _ := r.resourceContextControl.Get(newTargetContextDetail, v1alpha1.EnumReplaceOriginTargetIDContextDataKey) + if exist && originTargetId == strconv.Itoa(id) { mapNewToOriginTargetContext[id] = newTargetContextDetail } else { mapNewToOriginTargetContext[id] = nil @@ -386,13 +419,14 @@ func mapReplaceNewToOriginTargetContext(ownedIDs map[int]*api.ContextDetail) map return mapNewToOriginTargetContext } -func mapReplaceOriginToNewTargetContext(ownedIDs map[int]*api.ContextDetail) map[int]*api.ContextDetail { - mapOriginToNewTargetContext := make(map[int]*api.ContextDetail) +func (r *RealSyncControl) mapReplaceOriginToNewTargetContext(ownedIDs map[int]*v1alpha1.ContextDetail) map[int]*v1alpha1.ContextDetail { + mapOriginToNewTargetContext := make(map[int]*v1alpha1.ContextDetail) for id, contextDetail := range ownedIDs { - if val, exist := contextDetail.Data[ReplaceOriginTargetIDContextDataKey]; exist { + if val, exist := r.resourceContextControl.Get(contextDetail, v1alpha1.EnumReplaceOriginTargetIDContextDataKey); exist { originTargetId, _ := strconv.ParseInt(val, 10, 32) originTargetContextDetail, exist := ownedIDs[int(originTargetId)] - if exist && originTargetContextDetail.Data[ReplaceNewTargetIDContextDataKey] == strconv.Itoa(id) { + newTargetId, _ := r.resourceContextControl.Get(originTargetContextDetail, v1alpha1.EnumReplaceNewTargetIDContextDataKey) + if exist && newTargetId == strconv.Itoa(id) { mapOriginToNewTargetContext[id] = originTargetContextDetail } else { mapOriginToNewTargetContext[id] = nil diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 3481950..003f50a 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -22,18 +22,24 @@ import ( "sort" "strconv" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" clientutil "kusionstack.io/kube-utils/client" controllerutils "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" "kusionstack.io/kube-utils/xset/opslifecycle" ) -// getTargetsToDelete finds number of diff targets from filteredTargets to do scaleIn -func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, replaceMapping map[string]*targetWrapper, diff int) []*targetWrapper { +// getTargetsToDelete +// 1. finds number of diff targets from filteredPods to do scaleIn +// 2. finds targets allowed to scale in out of diff +func (r *RealSyncControl) getTargetsToDelete(xsetObject v1alpha1.XSetObject, filteredTargets []*targetWrapper, replaceMapping map[string]*targetWrapper, diff int) []*targetWrapper { var countedTargets []*targetWrapper for _, target := range filteredTargets { if _, exist := replaceMapping[target.GetName()]; exist { @@ -42,14 +48,21 @@ func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, r } // 1. select targets to delete in first round according to diff - sort.Sort(newActiveTargetsForDeletion(countedTargets)) + sort.Sort(newActiveTargetsForDeletion(countedTargets, r.xsetController.CheckReady, r.xsetController.GetReadyTime)) if diff > len(countedTargets) { diff = len(countedTargets) } // 2. select targets to delete in second round according to replace, delete, exclude var needDeleteTargets []*targetWrapper - for _, target := range countedTargets[:diff] { + for i, target := range countedTargets { + // find pods to be scaleIn out of diff, is allowed to ops + spec := r.xsetController.GetXSetSpec(xsetObject) + _, allowed := opslifecycle.AllowOps(r.updateConfig.xsetLabelAnnoMgr, r.scaleInLifecycleAdapter, ptr.Deref(spec.ScaleStrategy.OperationDelaySeconds, 0), target) + if i >= diff && !allowed { + continue + } + // don't scaleIn exclude target and its newTarget (if exist) if target.ToExclude { continue @@ -61,7 +74,7 @@ func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, r continue } // when scaleIn origin Target, newTarget should be deleted if not service available - if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); !serviceAvailable { + if !r.xsetController.CheckAvailable(target.Object) { needDeleteTargets = append(needDeleteTargets, replacePairTarget) } } @@ -72,12 +85,20 @@ func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, r } type ActiveTargetsForDeletion struct { - targets []*targetWrapper + targets []*targetWrapper + checkReadyFunc func(object client.Object) bool + getReadyTimeFunc func(object client.Object) *metav1.Time } -func newActiveTargetsForDeletion(targets []*targetWrapper) *ActiveTargetsForDeletion { +func newActiveTargetsForDeletion( + targets []*targetWrapper, + checkReadyFunc func(object client.Object) bool, + getReadyTimeFunc func(object client.Object) *metav1.Time, +) *ActiveTargetsForDeletion { return &ActiveTargetsForDeletion{ - targets: targets, + targets: targets, + checkReadyFunc: checkReadyFunc, + getReadyTimeFunc: getReadyTimeFunc, } } @@ -103,15 +124,26 @@ func (s *ActiveTargetsForDeletion) Less(i, j int) bool { return l.IsDuringScaleInOps } - // TODO consider service available timestamps + lReady, rReady := s.checkReadyFunc(l.Object), s.checkReadyFunc(r.Object) + if lReady != rReady { + return lReady + } - lCreationTime := l.Object.GetCreationTimestamp().Time - rCreationTime := r.Object.GetCreationTimestamp().Time - return lCreationTime.Before(rCreationTime) + if l.OpsPriority != nil && r.OpsPriority != nil { + if l.OpsPriority.PriorityClass != r.OpsPriority.PriorityClass { + return l.OpsPriority.PriorityClass < r.OpsPriority.PriorityClass + } + if l.OpsPriority.DeletionCost != r.OpsPriority.DeletionCost { + return l.OpsPriority.DeletionCost < r.OpsPriority.DeletionCost + } + } + + // TODO consider service available timestamps + return CompareTarget(l.Object, r.Object, s.checkReadyFunc, s.getReadyTimeFunc) } // doIncludeExcludeTargets do real include and exclude for targets which are allowed to in/exclude -func (r *RealSyncControl) doIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, excludeTargets, includeTargets []string, availableContexts []*api.ContextDetail) error { +func (r *RealSyncControl) doIncludeExcludeTargets(ctx context.Context, xset v1alpha1.XSetObject, excludeTargets, includeTargets []string, availableContexts []*v1alpha1.ContextDetail) error { var excludeErrs, includeErrs []error _, _ = controllerutils.SlowStartBatch(len(excludeTargets), controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { defer func() { excludeErrs = append(excludeErrs, err) }() @@ -125,30 +157,30 @@ func (r *RealSyncControl) doIncludeExcludeTargets(ctx context.Context, xset api. } // excludeTarget try to exclude a target from xset -func (r *RealSyncControl) excludeTarget(ctx context.Context, xsetObject api.XSetObject, targetName string) error { +func (r *RealSyncControl) excludeTarget(ctx context.Context, xsetObject v1alpha1.XSetObject, targetName string) error { target := r.xsetController.NewXObject() if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xsetObject.GetNamespace(), Name: targetName}, target); err != nil { return err } - target.GetLabels()[TargetOrphanedIndicateLabelKey] = "true" + r.xsetLabelAnnoMgr.Set(target.GetLabels(), api.XOrphanedIndicationLabelKey, "true") return r.xControl.OrphanTarget(xsetObject, target) } // includeTarget try to include a target into xset -func (r *RealSyncControl) includeTarget(ctx context.Context, xsetObject api.XSetObject, targetName, instanceId string) error { +func (r *RealSyncControl) includeTarget(ctx context.Context, xsetObject v1alpha1.XSetObject, targetName, instanceId string) error { target := r.xsetController.NewXObject() if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xsetObject.GetNamespace(), Name: targetName}, target); err != nil { return err } - target.GetLabels()[TargetInstanceIDLabelKey] = instanceId - delete(target.GetLabels(), TargetOrphanedIndicateLabelKey) + r.xsetLabelAnnoMgr.Set(target.GetLabels(), api.XInstanceIdLabelKey, instanceId) + r.xsetLabelAnnoMgr.Delete(target.GetLabels(), api.XOrphanedIndicationLabelKey) return r.xControl.AdoptTarget(xsetObject, target) } // reclaimScaleStrategy updates targetToDelete, targetToExclude, targetToInclude in scaleStrategy -func (r *RealSyncControl) reclaimScaleStrategy(ctx context.Context, deletedTargets, excludedTargets, includedTargets sets.String, xsetObject api.XSetObject) error { +func (r *RealSyncControl) reclaimScaleStrategy(ctx context.Context, deletedTargets, excludedTargets, includedTargets sets.String, xsetObject v1alpha1.XSetObject) error { xspec := r.xsetController.GetXSetSpec(xsetObject) // reclaim TargetToDelete toDeleteTargets := sets.NewString(xspec.ScaleStrategy.TargetToDelete...) @@ -162,7 +194,7 @@ func (r *RealSyncControl) reclaimScaleStrategy(ctx context.Context, deletedTarge toIncludeTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToInclude...) notIncludeTargets := toIncludeTargetNames.Delete(includedTargets.List()...) xspec.ScaleStrategy.TargetToInclude = notIncludeTargets.List() - if err := r.xsetController.UpdateScaleStrategy(xsetObject, &xspec.ScaleStrategy); err != nil { + if err := r.xsetController.UpdateScaleStrategy(ctx, r.Client, xsetObject, &xspec.ScaleStrategy); err != nil { return err } // update xsetObject.spec.scaleStrategy diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 253d738..e5591d5 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -39,6 +39,7 @@ import ( "kusionstack.io/kube-utils/controller/merge" controllerutils "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" "kusionstack.io/kube-utils/xset/opslifecycle" "kusionstack.io/kube-utils/xset/resourcecontexts" "kusionstack.io/kube-utils/xset/xcontrol" @@ -46,16 +47,16 @@ import ( const UnknownRevision = "__unknownRevision__" -func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, syncContext *SyncContext) []*targetUpdateInfo { +func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject v1alpha1.XSetObject, syncContext *SyncContext) []*targetUpdateInfo { activeTargets := FilterOutActiveTargetWrappers(syncContext.TargetWrappers) targetUpdateInfoList := make([]*targetUpdateInfo, len(activeTargets)) for i, target := range activeTargets { updateInfo := &targetUpdateInfo{ - targetWrapper: &syncContext.TargetWrappers[i], - InPlaceUpdateSupport: true, + targetWrapper: syncContext.TargetWrappers[i], } + // TODO decoration for target template updateInfo.UpdateRevision = syncContext.UpdatedRevision // decide this target current revision, or nil if not indicated if target.GetLabels() != nil { @@ -90,9 +91,8 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync spec := r.xsetController.GetXSetSpec(xsetObject) // decide whether the TargetOpsLifecycle is during ops or not - updateInfo.IsDuringOps = target.IsDuringUpdateOps - updateInfo.RequeueForOperationDelay, updateInfo.IsAllowOps = opslifecycle.AllowOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, RealValue(spec.UpdateStrategy.OperationDelaySeconds), target) - + updateInfo.RequeueForOperationDelay, updateInfo.IsAllowUpdateOps = opslifecycle.AllowOps(r.updateConfig.xsetLabelAnnoMgr, r.updateLifecycleAdapter, ptr.Deref(spec.UpdateStrategy.OperationDelaySeconds, 0), target) + // TODO check pvc template changed targetUpdateInfoList[i] = updateInfo } @@ -101,25 +101,30 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync for _, targetUpdateInfo := range targetUpdateInfoList { targetUpdateInfoMap[targetUpdateInfo.GetName()] = targetUpdateInfo } + // originTarget's isAllowUpdateOps depends on these 2 cases: + // (1) target is during replacing but not during replaceUpdate, keep it legacy value + // (2) target is during replaceUpdate, set to "true" if newTarget is service available for originTargetName, replacePairNewTarget := range syncContext.replacingMap { originTargetInfo := targetUpdateInfoMap[originTargetName] + _, replaceIndicated := r.xsetLabelAnnoMgr.Get(originTargetInfo.GetLabels(), api.XReplaceIndicationLabelKey) + _, replaceByReplaceUpdate := r.xsetLabelAnnoMgr.Get(originTargetInfo.GetLabels(), api.XReplaceByReplaceUpdateLabelKey) + isReplaceUpdating := replaceIndicated && replaceByReplaceUpdate + + originTargetInfo.IsInReplace = replaceIndicated + originTargetInfo.IsInReplaceUpdate = isReplaceUpdating + if replacePairNewTarget != nil { - originTargetInfo.IsInReplacing = true - // replace origin target not go through lifecycle, mark during ops manual - originTargetInfo.IsDuringOps = true - originTargetInfo.IsAllowOps = true + // origin target is allowed to ops if new pod is serviceAvailable + newTargetSa := r.xsetController.CheckAvailable(replacePairNewTarget.Object) + originTargetInfo.IsAllowUpdateOps = originTargetInfo.IsAllowUpdateOps || newTargetSa + // attach replace new target updateInfo ReplacePairNewTargetInfo := targetUpdateInfoMap[replacePairNewTarget.GetName()] - ReplacePairNewTargetInfo.IsInReplacing = true + ReplacePairNewTargetInfo.IsInReplace = true + // in case of to-replace label is removed from origin target, new target is still in replaceUpdate + ReplacePairNewTargetInfo.IsInReplaceUpdate = replaceByReplaceUpdate + ReplacePairNewTargetInfo.ReplacePairOriginTargetName = originTargetName originTargetInfo.ReplacePairNewTargetInfo = ReplacePairNewTargetInfo - } else { - _, replaceIndicated := originTargetInfo.GetLabels()[TargetReplaceIndicationLabelKey] - _, replaceByReplaceUpdate := originTargetInfo.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] - if replaceIndicated && replaceByReplaceUpdate { - originTargetInfo.IsInReplacing = true - originTargetInfo.IsDuringOps = true - originTargetInfo.IsAllowOps = true - } } } @@ -129,11 +134,10 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync continue } updateInfo := &targetUpdateInfo{ - targetWrapper: &target, - InPlaceUpdateSupport: true, - UpdateRevision: syncContext.UpdatedRevision, + targetWrapper: target, + UpdateRevision: syncContext.UpdatedRevision, } - if revision, exist := r.resourContextControl.Get(target.ContextDetail, api.EnumRevisionContextDataKey); exist && + if revision, exist := r.resourceContextControl.Get(target.ContextDetail, v1alpha1.EnumRevisionContextDataKey); exist && revision == syncContext.UpdatedRevision.GetName() { updateInfo.IsUpdatedRevision = true } @@ -154,46 +158,40 @@ func filterOutPlaceHolderUpdateInfos(targets []*targetUpdateInfo) []*targetUpdat return filteredTargetUpdateInfos } -func decideTargetToUpdate(xsetController api.XSetController, xset api.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { +func (r *RealSyncControl) decideTargetToUpdate(xsetController v1alpha1.XSetController, xset v1alpha1.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { spec := xsetController.GetXSetSpec(xset) + filteredPodInfos := r.getTargetsUpdateTargets(targetInfos) + if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByLabel != nil { - activeTargetInfos := filterOutPlaceHolderUpdateInfos(targetInfos) - return decideTargetToUpdateByLabel(xset, activeTargetInfos) + activeTargetInfos := filterOutPlaceHolderUpdateInfos(filteredPodInfos) + return r.decideTargetToUpdateByLabel(activeTargetInfos) } - return decideTargetToUpdateByPartition(xsetController, xset, targetInfos) + return r.decideTargetToUpdateByPartition(xsetController, xset, filteredPodInfos) } -func decideTargetToUpdateByLabel(_ api.XSetObject, targetInfos []*targetUpdateInfo) (targetToUpdate []*targetUpdateInfo) { +func (r *RealSyncControl) decideTargetToUpdateByLabel(targetInfos []*targetUpdateInfo) (targetToUpdate []*targetUpdateInfo) { for i := range targetInfos { - if _, exist := targetInfos[i].GetLabels()[XSetUpdateIndicateLabelKey]; exist { - // filter target which is in replace update and is the new created target - if targetInfos[i].IsInReplacing && targetInfos[i].ReplacePairOriginTargetName != "" { - continue - } + if _, exist := r.xsetLabelAnnoMgr.Get(targetInfos[i].GetLabels(), api.XSetUpdateIndicationLabelKey); exist { targetToUpdate = append(targetToUpdate, targetInfos[i]) continue } - // already in replace update. - if targetInfos[i].IsInReplacing && targetInfos[i].ReplacePairNewTargetInfo != nil { - targetToUpdate = append(targetToUpdate, targetInfos[i]) - continue - } + // TODO separate decoration and xset update progress } return targetToUpdate } -func decideTargetToUpdateByPartition(xsetController api.XSetController, xset api.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { +func (r *RealSyncControl) decideTargetToUpdateByPartition(xsetController v1alpha1.XSetController, xset v1alpha1.XSetObject, filteredTargetInfos []*targetUpdateInfo) []*targetUpdateInfo { spec := xsetController.GetXSetSpec(xset) replicas := ptr.Deref(spec.Replicas, 0) + partition := int32(0) if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByPartition != nil { partition = ptr.Deref(spec.UpdateStrategy.RollingUpdate.ByPartition.Partition, 0) } - filteredTargetInfos := getTargetsUpdateTargets(targetInfos) // update all or not update any replicas if partition == 0 { return filteredTargetInfos @@ -203,42 +201,44 @@ func decideTargetToUpdateByPartition(xsetController api.XSetController, xset api } // partial update replicas - ordered := newOrderedTargetUpdateInfos(filteredTargetInfos, xsetController.CheckReady) + ordered := newOrderedTargetUpdateInfos(filteredTargetInfos, xsetController.CheckReady, xsetController.GetReadyTime) sort.Sort(ordered) targetToUpdate := ordered.targets[:replicas-partition] return targetToUpdate } // when sort targets to choose update, only sort (1) replace origin targets, (2) non-exclude targets -func getTargetsUpdateTargets(targetInfos []*targetUpdateInfo) (filteredTargetInfos []*targetUpdateInfo) { +func (r *RealSyncControl) getTargetsUpdateTargets(targetInfos []*targetUpdateInfo) (filteredTargetInfos []*targetUpdateInfo) { for _, targetInfo := range targetInfos { - if targetInfo.IsInReplacing && targetInfo.ReplacePairOriginTargetName != "" { + if targetInfo.ReplacePairOriginTargetName != "" { continue } - if targetInfo.PlaceHolder { - _, isReplaceNewTarget := targetInfo.ContextDetail.Data[ReplaceOriginTargetIDContextDataKey] - _, isReplaceOriginTarget := targetInfo.ContextDetail.Data[ReplaceNewTargetIDContextDataKey] - if isReplaceNewTarget || isReplaceOriginTarget { + if _, isReplaceNewTarget := r.resourceContextControl.Get(targetInfo.ContextDetail, v1alpha1.EnumReplaceOriginTargetIDContextDataKey); isReplaceNewTarget { continue } } - filteredTargetInfos = append(filteredTargetInfos, targetInfo) } return filteredTargetInfos } -func newOrderedTargetUpdateInfos(targetInfos []*targetUpdateInfo, checkReadyFunc func(object client.Object) bool) *orderByDefault { +func newOrderedTargetUpdateInfos( + targetInfos []*targetUpdateInfo, + checkReadyFunc func(object client.Object) bool, + getReadyTimeFunc func(object client.Object) *metav1.Time, +) *orderByDefault { return &orderByDefault{ - targets: targetInfos, - checkReadyFunc: checkReadyFunc, + targets: targetInfos, + checkReadyFunc: checkReadyFunc, + getReadyTimeFunc: getReadyTimeFunc, } } type orderByDefault struct { - targets []*targetUpdateInfo - checkReadyFunc func(object client.Object) bool + targets []*targetUpdateInfo + checkReadyFunc func(object client.Object) bool + getReadyTimeFunc func(object client.Object) *metav1.Time } func (o *orderByDefault) Len() int { @@ -261,8 +261,8 @@ func (o *orderByDefault) Less(i, j int) bool { return true } - if l.IsDuringOps != r.IsDuringOps { - return l.IsDuringOps + if l.IsDuringUpdateOps != r.IsDuringUpdateOps { + return l.IsDuringUpdateOps } lReady, rReady := o.checkReadyFunc(l.Object), o.checkReadyFunc(r.Object) @@ -270,43 +270,50 @@ func (o *orderByDefault) Less(i, j int) bool { return lReady } - lCreationTime := l.Object.GetCreationTimestamp().Time - rCreationTime := r.Object.GetCreationTimestamp().Time - return lCreationTime.Before(rCreationTime) + if l.OpsPriority != nil && r.OpsPriority != nil { + if l.OpsPriority.PriorityClass != r.OpsPriority.PriorityClass { + return l.OpsPriority.PriorityClass < r.OpsPriority.PriorityClass + } + if l.OpsPriority.DeletionCost != r.OpsPriority.DeletionCost { + return l.OpsPriority.DeletionCost < r.OpsPriority.DeletionCost + } + } + + return CompareTarget(l.Object, r.Object, o.checkReadyFunc, o.getReadyTimeFunc) } type UpdateConfig struct { - xsetController api.XSetController - client client.Client - targetControl xcontrol.TargetControl - resourContextControl resourcecontexts.ResourceContextControl - recorder record.EventRecorder + xsetController v1alpha1.XSetController + xsetLabelAnnoMgr api.XSetLabelAnnotationManager + client client.Client + targetControl xcontrol.TargetControl + resourceContextControl resourcecontexts.ResourceContextControl + recorder record.EventRecorder - opsLifecycleMgr api.LifeCycleLabelManager - scaleInLifecycleAdapter api.LifecycleAdapter - updateLifecycleAdapter api.LifecycleAdapter + scaleInLifecycleAdapter v1alpha1.LifecycleAdapter + updateLifecycleAdapter v1alpha1.LifecycleAdapter cacheExpectations expectations.CacheExpectationsInterface targetGVK schema.GroupVersionKind } type TargetUpdater interface { - Setup(config *UpdateConfig, xset api.XSetObject) + Setup(config *UpdateConfig, xset v1alpha1.XSetObject) FulfillTargetUpdatedInfo(ctx context.Context, revision *appsv1.ControllerRevision, targetUpdateInfo *targetUpdateInfo) error BeginUpdateTarget(ctx context.Context, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (bool, error) - FilterAllowOpsTargets(ctx context.Context, targetToUpdate []*targetUpdateInfo, ownedIDs map[int]*api.ContextDetail, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (*time.Duration, error) + FilterAllowOpsTargets(ctx context.Context, targetToUpdate []*targetUpdateInfo, ownedIDs map[int]*v1alpha1.ContextDetail, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (*time.Duration, error) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error GetTargetUpdateFinishStatus(ctx context.Context, targetUpdateInfo *targetUpdateInfo) (bool, string, error) - FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error + FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error } type GenericTargetUpdater struct { - OwnerObject api.XSetObject + OwnerObject v1alpha1.XSetObject *UpdateConfig } -func (u *GenericTargetUpdater) Setup(config *UpdateConfig, xset api.XSetObject) { +func (u *GenericTargetUpdater) Setup(config *UpdateConfig, xset v1alpha1.XSetObject) { u.UpdateConfig = config u.OwnerObject = xset } @@ -316,9 +323,9 @@ func (u *GenericTargetUpdater) BeginUpdateTarget(_ context.Context, syncContext targetInfo := <-targetCh u.recorder.Eventf(targetInfo.Object, corev1.EventTypeNormal, "TargetUpdateLifecycle", "try to begin TargetOpsLifecycle for updating Target of XSet") - if updated, err := opslifecycle.BeginWithCleaningOld(u.opsLifecycleMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { + if updated, err := opslifecycle.BeginWithCleaningOld(u.xsetLabelAnnoMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { if !targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport { - return opslifecycle.WhenBeginDelete(u.opsLifecycleMgr, obj) + return opslifecycle.WhenBeginDelete(u.xsetLabelAnnoMgr, obj) } return false, nil }); err != nil { @@ -329,28 +336,27 @@ func (u *GenericTargetUpdater) BeginUpdateTarget(_ context.Context, syncContext return err } } - return nil }) updating := succCount > 0 if err != nil { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, err, "UpdateFailed", err.Error()) + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetUpdate, err, "UpdateFailed", err.Error()) return updating, err } else { - AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, nil, "Updated", "") + AddOrUpdateCondition(syncContext.NewStatus, v1alpha1.XSetUpdate, nil, "Updated", "") } return updating, nil } -func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candidates []*targetUpdateInfo, ownedIDs map[int]*api.ContextDetail, _ *SyncContext, targetCh chan *targetUpdateInfo) (*time.Duration, error) { +func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candidates []*targetUpdateInfo, ownedIDs map[int]*v1alpha1.ContextDetail, _ *SyncContext, targetCh chan *targetUpdateInfo) (*time.Duration, error) { var recordedRequeueAfter *time.Duration needUpdateContext := false for i := range candidates { targetInfo := candidates[i] if !targetInfo.PlaceHolder { - if !targetInfo.IsAllowOps { + if !targetInfo.IsAllowUpdateOps { continue } if targetInfo.RequeueForOperationDelay != nil { @@ -362,28 +368,28 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid } } - targetInfo.IsAllowOps = true + targetInfo.IsAllowUpdateOps = true if targetInfo.IsUpdatedRevision { continue } if _, exist := ownedIDs[targetInfo.ID]; !exist { - u.recorder.Eventf(u.OwnerObject, corev1.EventTypeWarning, "TargetBeforeUpdate", "target %s/%s is not allowed to update because cannot find context id %s in resourceContext", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetLabels()[TargetInstanceIDLabelKey]) + u.recorder.Eventf(u.OwnerObject, corev1.EventTypeWarning, "TargetBeforeUpdate", "target %s/%s is not allowed to update because cannot find context id %s in resourceContext", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetLabels()[u.xsetLabelAnnoMgr.Value(api.XInstanceIdLabelKey)]) continue } - if !u.resourContextControl.Contains(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) { + if !u.resourceContextControl.Contains(ownedIDs[targetInfo.ID], v1alpha1.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) { needUpdateContext = true - u.resourContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) + u.resourceContextControl.Put(ownedIDs[targetInfo.ID], v1alpha1.EnumRevisionContextDataKey, targetInfo.UpdateRevision.GetName()) } spec := u.xsetController.GetXSetSpec(u.OwnerObject) // mark targetContext "TargetRecreateUpgrade" if upgrade by recreate - isRecreateUpdatePolicy := spec.UpdateStrategy.UpdatePolicy == api.XSetRecreateTargetUpdateStrategyType + isRecreateUpdatePolicy := spec.UpdateStrategy.UpdatePolicy == v1alpha1.XSetRecreateTargetUpdateStrategyType if (!targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport) || isRecreateUpdatePolicy { - u.resourContextControl.Put(ownedIDs[targetInfo.ID], api.EnumRecreateUpdateContextDataKey, "true") + u.resourceContextControl.Put(ownedIDs[targetInfo.ID], v1alpha1.EnumRecreateUpdateContextDataKey, "true") } if targetInfo.PlaceHolder { @@ -397,15 +403,21 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(ctx context.Context, candid if needUpdateContext { u.recorder.Eventf(u.OwnerObject, corev1.EventTypeNormal, "UpdateToTargetContext", "try to update ResourceContext for XSet") err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return u.resourContextControl.UpdateToTargetContext(ctx, u.OwnerObject, ownedIDs) + return u.resourceContextControl.UpdateToTargetContext(ctx, u.OwnerObject, ownedIDs) }) return recordedRequeueAfter, err } return recordedRequeueAfter, nil } -func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *targetUpdateInfo) error { - if updated, err := opslifecycle.Finish(u.opsLifecycleMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { +func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error { + if finishByCancelUpdate { + // cancel update lifecycle + return opslifecycle.CancelOpsLifecycle(u.xsetLabelAnnoMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object) + } + + // target is ops finished, finish the lifecycle gracefully + if updated, err := opslifecycle.Finish(u.xsetLabelAnnoMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { return fmt.Errorf("failed to finish TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) } else if updated { // add an expectation for this target update, before next reconciling @@ -426,13 +438,13 @@ func RegisterInPlaceOnlyUpdater(targetUpdater TargetUpdater) { inPlaceOnlyTargetUpdater = targetUpdater } -func (r *RealSyncControl) newTargetUpdater(xset api.XSetObject) TargetUpdater { +func (r *RealSyncControl) newTargetUpdater(xset v1alpha1.XSetObject) TargetUpdater { spec := r.xsetController.GetXSetSpec(xset) var targetUpdater TargetUpdater switch spec.UpdateStrategy.UpdatePolicy { - case api.XSetRecreateTargetUpdateStrategyType: + case v1alpha1.XSetRecreateTargetUpdateStrategyType: targetUpdater = &recreateTargetUpdater{} - case api.XSetInPlaceOnlyTargetUpdateStrategyType: + case v1alpha1.XSetInPlaceOnlyTargetUpdateStrategyType: if inPlaceOnlyTargetUpdater != nil { targetUpdater = inPlaceOnlyTargetUpdater } else { @@ -440,7 +452,7 @@ func (r *RealSyncControl) newTargetUpdater(xset api.XSetObject) TargetUpdater { // implemented with InPlaceIfPossible policy as default for compatibility. targetUpdater = &inPlaceIfPossibleUpdater{} } - case api.XSetReplaceTargetUpdateStrategyType: + case v1alpha1.XSetReplaceTargetUpdateStrategyType: targetUpdater = &replaceUpdateTargetUpdater{} default: targetUpdater = &inPlaceIfPossibleUpdater{} @@ -465,14 +477,14 @@ type inPlaceIfPossibleUpdater struct { func (u *inPlaceIfPossibleUpdater) FulfillTargetUpdatedInfo(_ context.Context, revision *appsv1.ControllerRevision, targetUpdateInfo *targetUpdateInfo) error { // 1. build target from current and updated revision // TODO: use cache - currentTarget, err := NewTargetFrom(u.xsetController, u.OwnerObject, targetUpdateInfo.CurrentRevision, targetUpdateInfo.ID) + currentTarget, err := NewTargetFrom(u.xsetController, u.xsetLabelAnnoMgr, u.OwnerObject, targetUpdateInfo.CurrentRevision, targetUpdateInfo.ID) if err != nil { return fmt.Errorf("fail to build Target from current revision %s: %v", targetUpdateInfo.CurrentRevision.GetName(), err.Error()) } // TODO: use cache - UpdatedTarget, err := NewTargetFrom(u.xsetController, u.OwnerObject, targetUpdateInfo.UpdateRevision, targetUpdateInfo.ID) + UpdatedTarget, err := NewTargetFrom(u.xsetController, u.xsetLabelAnnoMgr, u.OwnerObject, targetUpdateInfo.UpdateRevision, targetUpdateInfo.ID) if err != nil { return fmt.Errorf("fail to build Target from updated revision %s: %v", targetUpdateInfo.UpdateRevision.GetName(), err.Error()) } @@ -530,7 +542,7 @@ func (u *inPlaceIfPossibleUpdater) GetTargetUpdateFinishStatus(_ context.Context } targetLastState := &TargetStatus{} - if lastStateJson, exist := targetUpdateInfo.GetAnnotations()[LastTargetStatusAnnotationKey]; !exist { + if lastStateJson, exist := u.xsetLabelAnnoMgr.Get(targetUpdateInfo.GetAnnotations(), api.LastXStatusAnnotationKey); !exist { return false, "no target last state annotation", nil } else if err := json.Unmarshal([]byte(lastStateJson), targetLastState); err != nil { msg := fmt.Sprintf("malformat target last state annotation [%s]: %s", lastStateJson, err.Error()) @@ -565,7 +577,7 @@ type replaceUpdateTargetUpdater struct { GenericTargetUpdater } -func (u *replaceUpdateTargetUpdater) Setup(config *UpdateConfig, xset api.XSetObject) { +func (u *replaceUpdateTargetUpdater) Setup(config *UpdateConfig, xset v1alpha1.XSetObject) { u.GenericTargetUpdater.Setup(config, xset) } @@ -578,7 +590,7 @@ func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, sync if exist && newTargetRevision == targetInfo.UpdateRevision.GetName() { return nil } - if _, exist := replacePairNewTarget.GetLabels()[TargetDeletionIndicationLabelKey]; exist { + if _, exist := u.xsetLabelAnnoMgr.Get(replacePairNewTarget.GetLabels(), api.XDeletionIndicationLabelKey); exist { return nil } @@ -590,7 +602,7 @@ func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, sync replacePairNewTarget.GetName(), newTargetRevision, syncContext.UpdatedRevision.GetName()) - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, u.xsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) if patchErr := u.client.Patch(ctx, targetInfo.ReplacePairNewTargetInfo.Object, patch); patchErr != nil { err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", targetInfo.ReplacePairNewTargetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), patchErr.Error()) @@ -603,7 +615,7 @@ func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, sync return succCount > 0, err } -func (u *replaceUpdateTargetUpdater) FilterAllowOpsTargets(_ context.Context, candidates []*targetUpdateInfo, _ map[int]*api.ContextDetail, _ *SyncContext, targetCh chan *targetUpdateInfo) (requeueAfter *time.Duration, err error) { +func (u *replaceUpdateTargetUpdater) FilterAllowOpsTargets(_ context.Context, candidates []*targetUpdateInfo, _ map[int]*v1alpha1.ContextDetail, _ *SyncContext, targetCh chan *targetUpdateInfo) (requeueAfter *time.Duration, err error) { activeTargetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) for i, targetInfo := range activeTargetToUpdate { if targetInfo.IsUpdatedRevision { @@ -620,25 +632,7 @@ func (u *replaceUpdateTargetUpdater) FulfillTargetUpdatedInfo(_ context.Context, } func (u *replaceUpdateTargetUpdater) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { - // add replace labels and wait to replace when syncTargets - _, replaceIndicate := targetInfo.Object.GetLabels()[TargetReplaceIndicationLabelKey] - _, replaceByUpdate := targetInfo.Object.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] - if !replaceIndicate || !replaceByUpdate { - // need replace update target, label target with replace-indicate and replace-update - now := time.Now().UnixNano() - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, TargetReplaceIndicationLabelKey, now, TargetReplaceByReplaceUpdateLabelKey, targetInfo.UpdateRevision.GetName()))) - if err := u.client.Patch(ctx, targetInfo.Object, patch); err != nil { - return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) - } - u.recorder.Eventf(targetInfo.Object, - corev1.EventTypeNormal, - "UpdateTarget", - "succeed to update Target %s/%s by label to-replace", - targetInfo.GetNamespace(), - targetInfo.GetName(), - ) - } - return nil + return updateReplaceOriginTarget(ctx, u.client, u.recorder, u.xsetLabelAnnoMgr, targetInfo, targetInfo.ReplacePairNewTargetInfo) } func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetUpdateInfo *targetUpdateInfo) (finished bool, msg string, err error) { @@ -650,11 +644,22 @@ func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Conte return u.isTargetUpdatedServiceAvailable(replaceNewTargetInfo) } -func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { +func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo, finishByCancelUpdate bool) error { + if finishByCancelUpdate { + // cancel replace update by removing to-replace and replace-by-update label from origin target + if targetInfo.IsInReplace { + patch := client.RawPatch(types.MergePatchType, fmt.Appendf(nil, `{"metadata":{"labels":{"%s":null, "%s":null}}}`, u.xsetLabelAnnoMgr.Value(api.XReplaceIndicationLabelKey), u.xsetLabelAnnoMgr.Value(api.XReplaceByReplaceUpdateLabelKey))) + if err := u.targetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { + return fmt.Errorf("failed to patch replace pair target %s/%s %w when cancel replace update", targetInfo.GetNamespace(), targetInfo.GetName(), err) + } + } + return nil + } + ReplacePairNewTargetInfo := targetInfo.ReplacePairNewTargetInfo if ReplacePairNewTargetInfo != nil { - if _, exist := targetInfo.GetLabels()[TargetDeletionIndicationLabelKey]; !exist { - patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + if _, exist := u.xsetLabelAnnoMgr.Get(targetInfo.GetLabels(), api.XDeletionIndicationLabelKey); !exist { + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, u.xsetLabelAnnoMgr.Value(api.XDeletionIndicationLabelKey), time.Now().UnixNano()))) if err := u.targetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { return fmt.Errorf("failed to delete replace pair origin target %s/%s %s", targetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), err.Error()) } @@ -664,14 +669,15 @@ func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, tar } func (u *GenericTargetUpdater) isTargetUpdatedServiceAvailable(targetInfo *targetUpdateInfo) (finished bool, msg string, err error) { + // TODO check decoration changed if targetInfo.GetLabels() == nil { return false, "no labels on target", nil } - if targetInfo.IsInReplacing && targetInfo.ReplacePairNewTargetInfo != nil { + if targetInfo.IsInReplace && targetInfo.ReplacePairNewTargetInfo != nil { return false, "replace origin target", nil } - if serviceAvailable := opslifecycle.IsServiceAvailable(u.opsLifecycleMgr, targetInfo.Object); serviceAvailable { + if u.xsetController.CheckAvailable(targetInfo.Object) { return true, "", nil } diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index 1cad1f6..83740cd 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -17,6 +17,7 @@ package synccontrols import ( + "context" "fmt" "strconv" "time" @@ -24,20 +25,23 @@ import ( appsv1 "k8s.io/api/apps/v1" apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "kusionstack.io/kube-api/apps/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" + clientutils "kusionstack.io/kube-utils/client" + controllerutils "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) -func GetInstanceID(target client.Object) (int, error) { +func GetInstanceID(xsetLabelAnnoMgr api.XSetLabelAnnotationManager, target client.Object) (int, error) { if target.GetLabels() == nil { return -1, fmt.Errorf("no labels found for instance ID") } - val, exist := target.GetLabels()[TargetInstanceIDLabelKey] + instanceIdLabelKey := xsetLabelAnnoMgr.Value(api.XInstanceIdLabelKey) + val, exist := target.GetLabels()[instanceIdLabelKey] if !exist { - return -1, fmt.Errorf("failed to find instance ID label %s", TargetInstanceIDLabelKey) + return -1, fmt.Errorf("failed to find instance ID label %s", instanceIdLabelKey) } id, err := strconv.ParseInt(val, 10, 32) @@ -49,7 +53,7 @@ func GetInstanceID(target client.Object) (int, error) { return int(id), nil } -func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revision *appsv1.ControllerRevision, id int, updateFuncs ...func(client.Object) error) (client.Object, error) { +func NewTargetFrom(setController v1alpha1.XSetController, xsetLabelAnnoMgr api.XSetLabelAnnotationManager, owner v1alpha1.XSetObject, revision *appsv1.ControllerRevision, id int, updateFuncs ...func(client.Object) error) (client.Object, error) { targetObj, err := setController.GetXObjectFromRevision(revision) if err != nil { return nil, err @@ -62,9 +66,9 @@ func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revis targetObj.SetGenerateName(GetTargetsPrefix(owner.GetName())) labels := targetObj.GetLabels() - labels[TargetInstanceIDLabelKey] = fmt.Sprintf("%d", id) + xsetLabelAnnoMgr.Set(labels, api.XInstanceIdLabelKey, fmt.Sprintf("%d", id)) labels[appsv1.ControllerRevisionHashLabelKey] = revision.GetName() - controlByKusionStack(targetObj) + controlByXSet(xsetLabelAnnoMgr, targetObj) for _, fn := range updateFuncs { if err := fn(targetObj); err != nil { @@ -75,17 +79,9 @@ func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revis return targetObj, nil } -func RealValue(val *int32) int32 { - if val == nil { - return 0 - } - - return *val -} - const ConditionUpdatePeriodBackOff = 30 * time.Second -func AddOrUpdateCondition(status *api.XSetStatus, conditionType api.XSetConditionType, err error, reason, message string) { +func AddOrUpdateCondition(status *v1alpha1.XSetStatus, conditionType v1alpha1.XSetConditionType, err error, reason, message string) { condStatus := metav1.ConditionTrue if err != nil { condStatus = metav1.ConditionFalse @@ -115,7 +111,7 @@ func NewCondition(condType string, status metav1.ConditionStatus, reason, msg st // SetCondition adds/replaces the given condition in the replicaset status. If the condition that we // are about to add already exists and has the same status and reason then we are not going to update. -func SetCondition(status *api.XSetStatus, condition *metav1.Condition) { +func SetCondition(status *v1alpha1.XSetStatus, condition *metav1.Condition) { currentCond := GetCondition(status, condition.Type) if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason && currentCond.LastTransitionTime == condition.LastTransitionTime { return @@ -126,7 +122,7 @@ func SetCondition(status *api.XSetStatus, condition *metav1.Condition) { } // GetCondition returns a inplace set condition with the provided type if it exists. -func GetCondition(status *api.XSetStatus, condType string) *metav1.Condition { +func GetCondition(status *v1alpha1.XSetStatus, condType string) *metav1.Condition { for _, c := range status.Conditions { if c.Type == condType { return &c @@ -171,12 +167,54 @@ func filterOutCondition(conditions []metav1.Condition, condType string) []metav1 return newConditions } -func controlByKusionStack(obj client.Object) { +func controlByXSet(xsetLabelAnnoMgr api.XSetLabelAnnotationManager, obj client.Object) { if obj.GetLabels() == nil { obj.SetLabels(map[string]string{}) } + if v, ok := xsetLabelAnnoMgr.Get(obj.GetLabels(), api.ControlledByXSetLabel); !ok || v != "true" { + xsetLabelAnnoMgr.Set(obj.GetLabels(), api.ControlledByXSetLabel, "true") + } +} + +func IsControlledByXSet(xsetLabelManager api.XSetLabelAnnotationManager, obj client.Object) bool { + if obj.GetLabels() == nil { + return false + } + + v, ok := xsetLabelManager.Get(obj.GetLabels(), api.ControlledByXSetLabel) + return ok && v == "true" +} + +func ApplyTemplatePatcher(ctx context.Context, xsetController v1alpha1.XSetController, c client.Client, xset v1alpha1.XSetObject, targets []*targetWrapper) error { + _, patchErr := controllerutils.SlowStartBatch(len(targets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + if targets[i].Object == nil || targets[i].PlaceHolder { + return nil + } + _, err := clientutils.UpdateOnConflict(ctx, c, c, targets[i].Object, xsetController.GetXSetTemplatePatcher(xset)) + return err + }) + return patchErr +} + +func CompareTarget(l, r client.Object, + checkReadyFunc func(object client.Object) bool, + getReadyTimeFunc func(object client.Object) *metav1.Time, +) bool { + // If both targets are ready, the latest ready one is smaller + if checkReadyFunc(l) && checkReadyFunc(r) && !getReadyTimeFunc(l).Equal(getReadyTimeFunc(r)) { + return afterOrZero(getReadyTimeFunc(l), getReadyTimeFunc(r)) + } + // Empty creation time targets < newer targets < older targets + lCreationTime, rCreationTime := l.GetCreationTimestamp(), r.GetCreationTimestamp() + if !(&lCreationTime).Equal(&rCreationTime) { + return afterOrZero(&lCreationTime, &rCreationTime) + } + return false +} - if v, ok := obj.GetLabels()[v1alpha1.ControlledByKusionStackLabelKey]; !ok || v != "true" { - obj.GetLabels()[v1alpha1.ControlledByKusionStackLabelKey] = "true" +func afterOrZero(t1, t2 *metav1.Time) bool { + if t1.Time.IsZero() || t2.Time.IsZero() { + return t1.Time.IsZero() } + return t1.After(t2.Time) } diff --git a/xset/xcontrol/target_control.go b/xset/xcontrol/target_control.go index 0c1661f..7e0dfbd 100644 --- a/xset/xcontrol/target_control.go +++ b/xset/xcontrol/target_control.go @@ -31,7 +31,7 @@ import ( "kusionstack.io/kube-utils/controller/mixin" refmanagerutil "kusionstack.io/kube-utils/controller/refmanager" - "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/api/v1alpha1" ) const ( @@ -39,24 +39,24 @@ const ( ) type TargetControl interface { - GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner api.XSetObject) ([]client.Object, error) + GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner v1alpha1.XSetObject) ([]client.Object, error) CreateTarget(ctx context.Context, target client.Object) (client.Object, error) DeleteTarget(ctx context.Context, target client.Object) error UpdateTarget(ctx context.Context, target client.Object) error PatchTarget(ctx context.Context, target client.Object, patch client.Patch) error - OrphanTarget(xset api.XSetObject, target client.Object) error - AdoptTarget(xset api.XSetObject, target client.Object) error + OrphanTarget(xset v1alpha1.XSetObject, target client.Object) error + AdoptTarget(xset v1alpha1.XSetObject, target client.Object) error } type targetControl struct { client client.Client schema *runtime.Scheme - xsetController api.XSetController + xsetController v1alpha1.XSetController xGVK schema.GroupVersionKind } -func NewTargetControl(mixin *mixin.ReconcilerMixin, xsetController api.XSetController) (TargetControl, error) { +func NewTargetControl(mixin *mixin.ReconcilerMixin, xsetController v1alpha1.XSetController) (TargetControl, error) { if err := setUpCache(mixin.Cache, xsetController); err != nil { return nil, err } @@ -71,7 +71,7 @@ func NewTargetControl(mixin *mixin.ReconcilerMixin, xsetController api.XSetContr }, nil } -func (r *targetControl) GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner api.XSetObject) ([]client.Object, error) { +func (r *targetControl) GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner v1alpha1.XSetObject) ([]client.Object, error) { targetList := r.xsetController.NewXObjectList() if err := r.client.List(ctx, targetList, &client.ListOptions{ Namespace: owner.GetNamespace(), @@ -121,7 +121,7 @@ func (r *targetControl) PatchTarget(ctx context.Context, target client.Object, p return r.client.Patch(ctx, target, patch) } -func (r *targetControl) OrphanTarget(xset api.XSetObject, target client.Object) error { +func (r *targetControl) OrphanTarget(xset v1alpha1.XSetObject, target client.Object) error { spec := r.xsetController.GetXSetSpec(xset) if spec.Selector.MatchLabels == nil { return nil @@ -142,7 +142,7 @@ func (r *targetControl) OrphanTarget(xset api.XSetObject, target client.Object) return nil } -func (r *targetControl) AdoptTarget(xset api.XSetObject, target client.Object) error { +func (r *targetControl) AdoptTarget(xset v1alpha1.XSetObject, target client.Object) error { spec := r.xsetController.GetXSetSpec(xset) if spec.Selector.MatchLabels == nil { return nil @@ -162,7 +162,7 @@ func (r *targetControl) AdoptTarget(xset api.XSetObject, target client.Object) e return nil } -func (r *targetControl) getTargets(candidates []client.Object, selector *metav1.LabelSelector, xset api.XSetObject) ([]client.Object, error) { +func (r *targetControl) getTargets(candidates []client.Object, selector *metav1.LabelSelector, xset v1alpha1.XSetObject) ([]client.Object, error) { // Use RefManager to adopt/orphan as needed. writer := refmanagerutil.NewOwnerRefWriter(r.client) matcher, err := refmanagerutil.LabelSelectorAsMatch(selector) @@ -187,7 +187,7 @@ func (r *targetControl) getTargets(candidates []client.Object, selector *metav1. return claimObjs, errors.Join(errList...) } -func setUpCache(cache cache.Cache, controller api.XSetController) error { +func setUpCache(cache cache.Cache, controller v1alpha1.XSetController) error { if err := cache.IndexField(context.TODO(), controller.NewXObject(), FieldIndexOwnerRefUID, func(object client.Object) []string { ownerRef := metav1.GetControllerOf(object) if ownerRef == nil || ownerRef.Kind != controller.XSetMeta().Kind { diff --git a/xset/xset_controller.go b/xset/xset_controller.go index 7653dd0..4f33569 100644 --- a/xset/xset_controller.go +++ b/xset/xset_controller.go @@ -22,6 +22,7 @@ import ( "fmt" "time" + "github.com/go-logr/logr" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -29,7 +30,9 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -38,7 +41,8 @@ import ( "kusionstack.io/kube-utils/controller/history" "kusionstack.io/kube-utils/controller/mixin" "kusionstack.io/kube-utils/xset/api" - "kusionstack.io/kube-utils/xset/api/validation" + "kusionstack.io/kube-utils/xset/api/v1alpha1" + "kusionstack.io/kube-utils/xset/api/v1alpha1/validation" "kusionstack.io/kube-utils/xset/resourcecontexts" "kusionstack.io/kube-utils/xset/revisionowner" "kusionstack.io/kube-utils/xset/synccontrols" @@ -48,7 +52,7 @@ import ( type xSetCommonReconciler struct { mixin.ReconcilerMixin - XSetController api.XSetController + XSetController v1alpha1.XSetController meta metav1.TypeMeta finalizerName string xsetGVK schema.GroupVersionKind @@ -61,13 +65,21 @@ type xSetCommonReconciler struct { resourceContextControl resourcecontexts.ResourceContextControl } -func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error { - // TODO: validate xsetController +func SetUpWithManager(mgr ctrl.Manager, xsetController v1alpha1.XSetController) error { + err := validation.ValidateXSetController(xsetController) + if err != nil { + return err + } resourceContextAdapter, err := getAndValidateResourceContextAdapter(xsetController) if err != nil { return err } + xsetLabelManager := xsetController.GetXSetControllerLabelManager() + if xsetLabelManager == nil { + xsetLabelManager = api.NewXSetLabelAnnotationManager() + } + reconcilerMixin := mixin.NewReconcilerMixin(xsetController.ControllerName(), mgr) xsetMeta := xsetController.XSetMeta() xsetGVK := xsetMeta.GroupVersionKind() @@ -81,7 +93,7 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error } cacheExpectations := expectations.NewxCacheExpectations(reconcilerMixin.Client, reconcilerMixin.Scheme, clock.RealClock{}) resourceContextControl := resourcecontexts.NewRealResourceContextControl(reconcilerMixin.Client, xsetController, resourceContextAdapter, resourceContextGVK, cacheExpectations) - syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, resourceContextControl, cacheExpectations) + syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, xsetLabelManager, resourceContextControl, cacheExpectations) revisionControl := history.NewRevisionControl(reconcilerMixin.Client, reconcilerMixin.Client) revisionOwner := revisionowner.NewRevisionOwner(xsetController, targetControl) revisionManager := history.NewHistoryManager(revisionControl, revisionOwner) @@ -114,6 +126,20 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error if err := c.Watch(&source.Kind{Type: xsetController.NewXObject()}, &handler.EnqueueRequestForOwner{ IsController: true, OwnerType: xsetController.NewXSetObject(), + }, predicate.Funcs{ + CreateFunc: func(event event.CreateEvent) bool { + return synccontrols.IsControlledByXSet(xsetLabelManager, event.Object) + }, + UpdateFunc: func(updateEvent event.UpdateEvent) bool { + return synccontrols.IsControlledByXSet(xsetLabelManager, updateEvent.ObjectNew) || + synccontrols.IsControlledByXSet(xsetLabelManager, updateEvent.ObjectOld) + }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { + return synccontrols.IsControlledByXSet(xsetLabelManager, deleteEvent.Object) + }, + GenericFunc: func(genericEvent event.GenericEvent) bool { + return synccontrols.IsControlledByXSet(xsetLabelManager, genericEvent.Object) + }, }); err != nil { return fmt.Errorf("failed to watch %s: %s", targetMeta.Kind, err.Error()) } @@ -121,7 +147,7 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error return nil } -func getAndValidateResourceContextAdapter(xsetController api.XSetController) (api.ResourceContextAdapter, error) { +func getAndValidateResourceContextAdapter(xsetController v1alpha1.XSetController) (v1alpha1.ResourceContextAdapter, error) { resourceContextAdapter := xsetController.GetResourceContextAdapter() if resourceContextAdapter == nil { resourceContextAdapter = &resourcecontexts.DefaultResourceContextAdapter{} @@ -135,7 +161,8 @@ func getAndValidateResourceContextAdapter(xsetController api.XSetController) (ap func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { kind := r.meta.Kind key := req.String() - logger := r.Logger.WithValues(kind, key) + ctx = logr.NewContext(ctx, r.Logger.WithValues(kind, key)) + logger := logr.FromContext(ctx) instance := r.XSetController.NewXSetObject() if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil { if !apierrors.IsNotFound(err) { @@ -206,7 +233,7 @@ func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Requ return requeueResult(requeueAfter), syncErr } -func (r *xSetCommonReconciler) doSync(ctx context.Context, instance api.XSetObject, syncContext *synccontrols.SyncContext) (*time.Duration, error) { +func (r *xSetCommonReconciler) doSync(ctx context.Context, instance v1alpha1.XSetObject, syncContext *synccontrols.SyncContext) (*time.Duration, error) { synced, err := r.syncControl.SyncTargets(ctx, instance, syncContext) if err != nil || synced { return nil, err @@ -219,24 +246,25 @@ func (r *xSetCommonReconciler) doSync(ctx context.Context, instance api.XSetObje _, scaleRequeueAfter, scaleErr := r.syncControl.Scale(ctx, instance, syncContext) _, updateRequeueAfter, updateErr := r.syncControl.Update(ctx, instance, syncContext) + patcherErr := synccontrols.ApplyTemplatePatcher(ctx, r.XSetController, r.Client, instance, syncContext.TargetWrappers) - err = errors.Join(scaleErr, updateErr) + err = errors.Join(scaleErr, updateErr, patcherErr) if updateRequeueAfter != nil && (scaleRequeueAfter == nil || *updateRequeueAfter < *scaleRequeueAfter) { return updateRequeueAfter, err } return scaleRequeueAfter, err } -func (r *xSetCommonReconciler) ensureReclaimTargetsDeletion(ctx context.Context, instance api.XSetObject) error { +func (r *xSetCommonReconciler) ensureReclaimTargetsDeletion(ctx context.Context, instance v1alpha1.XSetObject) error { xSetSpec := r.XSetController.GetXSetSpec(instance) targets, err := r.targetControl.GetFilteredTargets(ctx, xSetSpec.Selector, instance) if err != nil { return fmt.Errorf("fail to get filtered Targets: %s", err.Error()) } - return synccontrols.BatchDeleteTargetByLabel(ctx, r.targetControl, targets) + return r.syncControl.BatchDeleteTargetsByLabel(ctx, r.targetControl, targets) } -func (r *xSetCommonReconciler) updateStatus(ctx context.Context, instance api.XSetObject, status *api.XSetStatus) error { +func (r *xSetCommonReconciler) updateStatus(ctx context.Context, instance v1alpha1.XSetObject, status *v1alpha1.XSetStatus) error { r.XSetController.SetXSetStatus(instance, status) if err := r.Client.Status().Update(ctx, instance); err != nil { return fmt.Errorf("fail to update status of %s: %s", instance.GetName(), err.Error())