Fix issue with injector webhook applying bindings from all namespaces instead of selected, update manifests to not use auth-proxy

Author: gbraxton

README.md

@@ -46,6 +46,8 @@ Aside from defining simple key/value pairs, Kconfigs can also define and referen
 
 For a target to have its environment variables controlled by Kconfigs, it needs the annotation ```kconfigcontroller.atteg.com/inject=true```.
 
+Add the annotation, ```kconfigcontroller.atteg.com/refresh-template=true``` to have updates to a kconfig to trigger a rolling update for deployments, statefulsets of the selected pods.
+
 Kconfig-controller also has a secondary custom resource, KconfigBinding, that is used by the controllers and should not be created/manipulated directly by users. This resources serve as a target for Kconfigs to update their changes whereafter, the admission-controller can import the contained environment variables directly into pods. Note that there is a one-to-one mapping for each kconfig and kconfigbinding.
 
 Build requires Kustomize (https://github.com/kubernetes-sigs/kustomize) locally and cert-manager (https://github.com/jetstack/cert-manager) installed in the kubernetes cluser for the admission-controller's TLS certificates.

api/v1beta1/kconfig_types.go

@@ -25,20 +25,27 @@ import (
 
 // KconfigSpec defines the desired state of Kconfig
 type KconfigSpec struct {
-	Level      int                  `json:"level"`
+	Level int `json:"level"`
+	// +kubebuilder:validation:Optional
 	Selector   metav1.LabelSelector `json:"selector"`
 	EnvConfigs []EnvConfig          `json:"envConfigs"`
 }
 
 // EnvConfig represents a single environment variable configuration
 type EnvConfig struct {
 	// Type should be immutable
-	Type             string                    `json:"type"`
-	Key              string                    `json:"key"`
-	Value            *string                   `json:"value,omitempty"`
-	ConfigMapKeyRef  *v1.ConfigMapKeySelector  `json:"configMapKeyRef,omitempty"`
-	SecretKeyRef     *v1.SecretKeySelector     `json:"secretKeyRef,omitempty" protobuf:"bytes,4,opt,name=secretKeyRef"`
-	FieldRef         *v1.ObjectFieldSelector   `json:"fieldRef,omitempty" protobuf:"bytes,4,opt,name=fieldRef"`
+	// +kubebuilder:validation:Optional
+	Type string `json:"type"`
+	Key  string `json:"key"`
+	// +kubebuilder:validation:Optional
+	Value *string `json:"value,omitempty"`
+	// +kubebuilder:validation:Optional
+	ConfigMapKeyRef *v1.ConfigMapKeySelector `json:"configMapKeyRef,omitempty"`
+	// +kubebuilder:validation:Optional
+	SecretKeyRef *v1.SecretKeySelector `json:"secretKeyRef,omitempty" protobuf:"bytes,4,opt,name=secretKeyRef"`
+	// +kubebuilder:validation:Optional
+	FieldRef *v1.ObjectFieldSelector `json:"fieldRef,omitempty" protobuf:"bytes,4,opt,name=fieldRef"`
+	// +kubebuilder:validation:Optional
 	ResourceFieldRef *v1.ResourceFieldSelector `json:"resourceFieldRef,omitempty" protobuf:"bytes,4,opt,name=resourceFieldRef"`
 }
 

api/v1beta1/kconfigbinding_types.go

@@ -25,8 +25,9 @@ import (
 
 // KconfigBindingSpec defines the desired state of KconfigBinding
 type KconfigBindingSpec struct {
-	Level    int                  `json:"level"`
-	Envs     []v1.EnvVar          `json:"envs"`
+	Level int         `json:"level"`
+	Envs  []v1.EnvVar `json:"envs"`
+	// +kubebuilder:validation:Optional
 	Selector metav1.LabelSelector `json:"selector"`
 }
 

config/crd/bases/kconfigcontroller.atteg.com_kconfigbindings.yaml

@@ -184,7 +184,6 @@ spec:
           required:
           - envs
           - level
-          - selector
           type: object
         status:
           description: KconfigBindingStatus defines the observed state of KconfigBinding

config/crd/bases/kconfigcontroller.atteg.com_kconfigs.yaml

@@ -115,7 +115,6 @@ spec:
                     type: string
                 required:
                 - key
-                - type
                 type: object
               type: array
             level:
@@ -169,7 +168,6 @@ spec:
           required:
           - envConfigs
           - level
-          - selector
           type: object
         status:
           description: KconfigStatus defines the observed state of Kconfig

config/default/kustomization.yaml

@@ -27,7 +27,7 @@ patchesStrategicMerge:
   # Protect the /metrics endpoint by putting it behind auth.
   # Only one of manager_auth_proxy_patch.yaml and
   # manager_prometheus_metrics_patch.yaml should be enabled.
-- manager_auth_proxy_patch.yaml
+#- manager_auth_proxy_patch.yaml
   # If you want your controller-manager to expose the /metrics
   # endpoint w/o any authn/z, uncomment the following line and
   # comment manager_auth_proxy_patch.yaml.

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: docker-registry.aeg.cloud/kcnfig-system/kconfig-controller
-  newTag: ref-202002231749
+  newName: docker-registry.aeg.cloud/kconfig-system/kconfig-controller
+  newTag: v1beta1-202002261210

config/manager/manager.yaml

@@ -21,19 +21,25 @@ spec:
     metadata:
       labels:
         control-plane: controller-manager
+      annotations:
+        prometheus.io/path: /metrics
+        prometheus.io/port: "8080"
+        prometheus.io/scheme: https
+        prometheus.io/scrape: "true"
     spec:
       containers:
       - command:
         - /manager
         args:
+        - --metrics-addr=:8080
         - --enable-leader-election
         image: controller:latest
         name: manager
         resources:
           limits:
-            cpu: 100m
-            memory: 30Mi
+            cpu: "4"
+            memory: 6Gi
           requests:
-            cpu: 100m
-            memory: 20Mi
+            cpu: "4"
+            memory: 6Gi
       terminationGracePeriodSeconds: 10

config/rbac/kustomization.yaml

@@ -6,6 +6,6 @@ resources:
 # Comment the following 3 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
+#- auth_proxy_service.yaml
+#- auth_proxy_role.yaml
+#- auth_proxy_role_binding.yaml

config/rbac/role.yaml

@@ -6,6 +6,30 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  - statefulsets
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - kconfigcontroller.atteg.com
   resources:

config/webhook/manifests.yaml

@@ -12,7 +12,7 @@ webhooks:
       name: webhook-service
       namespace: system
       path: /mutate-v1-pod
-  failurePolicy: Fail
+  failurePolicy: Ignore
   name: config-injector.kconfigcontroller.aeg.cloud
   rules:
   - apiGroups:

controllers/const.go

@@ -4,11 +4,12 @@ const (
 	WarningEventType      = "Warning"
 	InvalidEnvConfigEvent = "InvalidEnvConfig"
 
+	ValueEnvConfigType            = "Value"
 	ConfigMapEnvConfigType        = "ConfigMap"
 	SecretEnvConfigType           = "Secret"
 	FieldRefEnvConfigType         = "FieldRef"
 	ResourceFieldRefEnvConfigType = "ResourceFieldRef"
 
-	AllowTemplateUpdatesAnnotation = "kconfigcontroller.atteg.com/update-template"
+	AllowTemplateUpdatesAnnotation = "kconfigcontroller.atteg.com/refresh-template"
 	GenerationAnnotationPrefix     = "kconfigcontroller.atteg.com/"
 )

controllers/kconfig_controller.go

@@ -45,6 +45,7 @@ type KconfigReconciler struct {
 
 // +kubebuilder:rbac:groups=kconfigcontroller.atteg.com,resources=kconfigs,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=kconfigcontroller.atteg.com,resources=kconfigs/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups="",resources=configmaps;secrets,verbs=get;list;watch;create;update;patch;delete
 
 func (r *KconfigReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	ctx := context.Background()
@@ -129,9 +130,10 @@ func (r *KconfigReconciler) processKconfig(ctx context.Context, kc *kconfigcontr
 func (r *KconfigReconciler) processValueEnvConfig(ec kconfigcontrollerv1beta1.EnvConfig, envVars *[]v1.EnvVar, updatedECs *[]kconfigcontrollerv1beta1.EnvConfig) error {
 	if ec.Key == "" || ec.Value == nil {
 		r.Recorder.Event(&kconfigcontrollerv1beta1.Kconfig{}, WarningEventType, InvalidEnvConfigEvent, "Either key or value is empty for value type EnvConfig. This entry will be removed")
+		return nil
 	}
 	*envVars = append(*envVars, v1.EnvVar{Name: ec.Key, Value: *ec.Value})
-	*updatedECs = append(*updatedECs, *ec.DeepCopy())
+	*updatedECs = append(*updatedECs, kconfigcontrollerv1beta1.EnvConfig{Type: ValueEnvConfigType, Key: ec.Key, Value: ec.Value})
 	return nil
 }
 

main.go

@@ -95,7 +95,7 @@ func main() {
 	hookServer.Register("/mutate-v1-pod", &webhook.Admission{
 		Handler: &webhooks.PodConfigInjector{
 			Client: mgr.GetClient(),
-			Log: ctrl.Log.WithName("webhooks").WithName("pod-config-injector"),
+			Log:    ctrl.Log.WithName("webhooks").WithName("pod-config-injector"),
 		},
 	})
 

webhooks/pod_injector_webhook.go

@@ -35,12 +35,12 @@ const (
 	InjectConfigAnnotation = "kconfigcontroller.atteg.com/inject"
 )
 
-// +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create,versions=v1,name=config-injector.kconfigcontroller.aeg.cloud
+// +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=ignore,groups="",resources=pods,verbs=create,versions=v1,name=config-injector.kconfigcontroller.aeg.cloud
 
 type PodConfigInjector struct {
 	Client  client.Client
 	decoder *admission.Decoder
-	Log             logr.Logger
+	Log     logr.Logger
 }
 
 func (r *PodConfigInjector) InjectDecoder(d *admission.Decoder) error {
@@ -50,6 +50,7 @@ func (r *PodConfigInjector) InjectDecoder(d *admission.Decoder) error {
 
 func (r *PodConfigInjector) Handle(ctx context.Context, req admission.Request) admission.Response {
 	pod := &v1.Pod{}
+
 	err := r.decoder.Decode(req, pod)
 	if err != nil {
 		return admission.Errored(http.StatusBadRequest, err)
@@ -61,7 +62,7 @@ func (r *PodConfigInjector) Handle(ctx context.Context, req admission.Request) a
 	}
 	// get bindings that select this rs
 	kcbs := v1beta1.KconfigBindingList{}
-	if err := r.Client.List(ctx, &kcbs, client.InNamespace(pod.Namespace)); err != nil {
+	if err := r.Client.List(ctx, &kcbs, client.InNamespace(req.Namespace)); err != nil {
 		return admission.Errored(http.StatusInternalServerError, fmt.Errorf("could not get kconfigbininglist: %s", err.Error()))
 	}