Store cache in secret to avoid added conversion

Signed-off-by: Danil-Grigorev <[email protected]>

Author: Danil-Grigorev

api/v1alpha2/provider_types.go

@@ -30,6 +30,7 @@ const (
 	ConfigMapNameLabel        = "provider.cluster.x-k8s.io/name"
 
 	CompressedAnnotation = "provider.cluster.x-k8s.io/compressed"
+	TrueValue            = "true"
 
 	MetadataConfigMapKey            = "metadata"
 	ComponentsConfigMapKey          = "components"

cmd/main.go

@@ -40,10 +40,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
-
 	ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
@@ -176,9 +174,6 @@ func main() {
 				},
 			},
 		},
-		Controller: config.Controller{
-			MaxConcurrentReconciles: concurrencyNumber,
-		},
 		WebhookServer: ctrlwebhook.NewServer(
 			ctrlwebhook.Options{
 				Port:    webhookPort,

internal/controller/genericprovider_controller.go

@@ -28,13 +28,13 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/rest"
 	operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
 	"sigs.k8s.io/cluster-api-operator/internal/controller/genericprovider"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	configclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
-	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/yamlprocessor"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/cluster-api/util/patch"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -44,8 +44,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-
-	utilyaml "sigs.k8s.io/cluster-api/util/yaml"
 )
 
 type GenericProviderReconciler struct {
@@ -335,14 +333,14 @@ func calculateHash(ctx context.Context, k8sClient client.Client, provider generi
 func applyFromCache(ctx context.Context, cl client.Client, provider genericprovider.GenericProvider) (bool, error) {
 	log := log.FromContext(ctx)
 
-	configMap := &corev1.ConfigMap{}
-	if err := cl.Get(ctx, client.ObjectKey{Name: ProviderCacheName(provider), Namespace: provider.GetNamespace()}, configMap); apierrors.IsNotFound(err) {
-		// config map does not exist, nothing to apply
+	secret := &corev1.Secret{}
+	if err := cl.Get(ctx, client.ObjectKey{Name: ProviderCacheName(provider), Namespace: provider.GetNamespace()}, secret); apierrors.IsNotFound(err) {
+		// secret does not exist, nothing to apply
 		return false, nil
 	} else if err != nil {
-		log.Error(err, "failed to get provider config map")
+		log.Error(err, "failed to get provider cache")
 
-		return false, fmt.Errorf("failed to get cache ConfigMap: %w", err)
+		return false, fmt.Errorf("failed to get provider cache: %w", err)
 	}
 
 	// calculate combined hash for provider and config map cache
@@ -353,15 +351,15 @@ func applyFromCache(ctx context.Context, cl client.Client, provider genericprovi
 		return false, err
 	}
 
-	if err := addObjectToHash(hash, configMap.Data); err != nil {
+	if err := addObjectToHash(hash, secret.Data); err != nil {
 		log.Error(err, "failed to calculate config map hash")
 
 		return false, err
 	}
 
 	cacheHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if configMap.GetAnnotations()[appliedSpecHashAnnotation] != cacheHash {
-		log.Info("Provider or cache state has changed", "cacheHash", cacheHash, "providerHash", configMap.GetAnnotations()[appliedSpecHashAnnotation])
+	if secret.GetAnnotations()[appliedSpecHashAnnotation] != cacheHash {
+		log.Info("Provider or cache state has changed", "cacheHash", cacheHash, "providerHash", secret.GetAnnotations()[appliedSpecHashAnnotation])
 
 		return false, nil
 	}
@@ -381,63 +379,52 @@ func applyFromCache(ctx context.Context, cl client.Client, provider genericprovi
 		return false, err
 	}
 
-	processor := yamlprocessor.NewSimpleProcessor()
-	for _, manifest := range configMap.Data {
-		manifest, err := processor.Process([]byte(manifest), mr.Get)
-		if err != nil {
-			log.Error(err, "failed to process manifest")
-
-			return false, err
+	for _, manifest := range secret.Data {
+		if secret.GetAnnotations()[operatorv1.CompressedAnnotation] == operatorv1.TrueValue {
+			break
 		}
 
-		manifests, err := utilyaml.ToUnstructured(manifest)
+		manifests := []unstructured.Unstructured{}
+
+		err := json.Unmarshal(manifest, &manifests)
 		if err != nil {
 			log.Error(err, "failed to convert yaml to unstructured")
 
 			return false, err
 		}
 
-		if len(manifests) > 1 {
-			return false, fmt.Errorf("multiple manifests found: %d", len(manifests))
-		} else if len(manifests) == 0 {
-			continue
+		for _, manifest := range manifests {
+			if err := cl.Patch(ctx, &manifest, client.Apply, client.ForceOwnership, client.FieldOwner(cacheOwner)); err != nil {
+				errs = append(errs, err)
+			}
 		}
+	}
 
-		if err := cl.Patch(ctx, &manifests[0], client.Apply, client.ForceOwnership, client.FieldOwner(cacheOwner)); err != nil {
-			errs = append(errs, err)
+	for _, binaryManifest := range secret.Data {
+		if secret.GetAnnotations()[operatorv1.CompressedAnnotation] != operatorv1.TrueValue {
+			break
 		}
-	}
 
-	for _, binaryManifest := range configMap.BinaryData {
-		manifest, err := decompressYaml(binaryManifest)
+		manifest, err := decompressData(binaryManifest)
 		if err != nil {
 			log.Error(err, "failed to decompress yaml")
 
 			return false, err
 		}
 
-		manifest, err = processor.Process(manifest, mr.Get)
-		if err != nil {
-			log.Error(err, "failed to process manifest")
-
-			return false, err
-		}
+		manifests := []unstructured.Unstructured{}
 
-		manifests, err := utilyaml.ToUnstructured(manifest)
+		err = json.Unmarshal(manifest, &manifests)
 		if err != nil {
 			log.Error(err, "failed to convert yaml to unstructured")
 
 			return false, err
 		}
 
-		if len(manifests) > 1 {
-			return false, fmt.Errorf("multiple manifests found: %d", len(manifests))
-		} else if len(manifests) == 0 {
-			continue
-		}
-
-		if err := cl.Patch(ctx, &manifests[0], client.Apply, client.ForceOwnership, client.FieldOwner(cacheOwner)); err != nil {
-			errs = append(errs, err)
+		for _, manifest := range manifests {
+			if err := cl.Patch(ctx, &manifest, client.Apply, client.ForceOwnership, client.FieldOwner(cacheOwner)); err != nil {
+				errs = append(errs, err)
+			}
 		}
 	}
 
@@ -452,14 +439,14 @@ func applyFromCache(ctx context.Context, cl client.Client, provider genericprovi
 	return true, nil
 }
 
-// setCacheHash calculates current provider and configMap hash, and updates it on the configMap.
+// setCacheHash calculates current provider and secret hash, and updates it on the secret.
 func setCacheHash(ctx context.Context, cl client.Client, provider genericprovider.GenericProvider) error {
-	configMap := &corev1.ConfigMap{}
-	if err := cl.Get(ctx, client.ObjectKey{Name: ProviderCacheName(provider), Namespace: provider.GetNamespace()}, configMap); err != nil {
-		return fmt.Errorf("failed to get cache ConfigMaps: %w", err)
+	secret := &corev1.Secret{}
+	if err := cl.Get(ctx, client.ObjectKey{Name: ProviderCacheName(provider), Namespace: provider.GetNamespace()}, secret); err != nil {
+		return fmt.Errorf("failed to get cache secret: %w", err)
 	}
 
-	helper, err := patch.NewHelper(configMap, cl)
+	helper, err := patch.NewHelper(secret, cl)
 	if err != nil {
 		return err
 	}
@@ -470,19 +457,19 @@ func setCacheHash(ctx context.Context, cl client.Client, provider genericprovide
 		return err
 	}
 
-	if err := addObjectToHash(hash, configMap.Data); err != nil {
+	if err := addObjectToHash(hash, secret.Data); err != nil {
 		return err
 	}
 
 	cacheHash := fmt.Sprintf("%x", hash.Sum(nil))
 
-	annotations := configMap.GetAnnotations()
+	annotations := secret.GetAnnotations()
 	if annotations == nil {
 		annotations = map[string]string{}
 	}
 
 	annotations[appliedSpecHashAnnotation] = cacheHash
-	configMap.SetAnnotations(annotations)
+	secret.SetAnnotations(annotations)
 
-	return helper.Patch(ctx, configMap)
+	return helper.Patch(ctx, secret)
 }

internal/controller/manifests_downloader.go

@@ -191,7 +191,7 @@ func TemplateManifestsConfigMap(provider operatorv1.GenericProvider, labels map[
 		configMap.Data[operatorv1.ComponentsConfigMapKey] = string(components)
 	} else {
 		var componentsBuf bytes.Buffer
-		if err := compressYaml(&componentsBuf, components); err != nil {
+		if err := compressData(&componentsBuf, components); err != nil {
 			return nil, fmt.Errorf("cannot compress data for provider %s/%s: %w", provider.GetNamespace(), provider.GetName(), err)
 		}
 
@@ -217,8 +217,8 @@ func TemplateManifestsConfigMap(provider operatorv1.GenericProvider, labels map[
 	return configMap, nil
 }
 
-// compressYaml takes a bytes.Buffer and data, and compresses data into it.
-func compressYaml(componentsBuf *bytes.Buffer, data []byte) (err error) {
+// compressData takes a bytes.Buffer and data, and compresses data into it.
+func compressData(componentsBuf *bytes.Buffer, data []byte) (err error) {
 	zw := gzip.NewWriter(componentsBuf)
 
 	_, err = zw.Write(data)
@@ -233,8 +233,8 @@ func compressYaml(componentsBuf *bytes.Buffer, data []byte) (err error) {
 	return
 }
 
-// decompressYaml takes a compressed data, and decompresses it.
-func decompressYaml(compressedData []byte) (data []byte, err error) {
+// decompressData takes a compressed data, and decompresses it.
+func decompressData(compressedData []byte) (data []byte, err error) {
 	zr, err := gzip.NewReader(bytes.NewReader(compressedData))
 	if err != nil {
 		return nil, fmt.Errorf("cannot open gzip reader from data: %w", err)