WIP: MON-4256: CHANGE: Test prometheus-operator resources moving to EndpointSlice

assets/node-exporter/daemonset.yaml

@@ -32,9 +32,10 @@ spec:
       automountServiceAccountToken: true
       containers:
       - args:
-        - --web.listen-address=127.0.0.1:9100
+        - --web.listen-address=127.0.0.1:9101
         - --path.sysfs=/host/sys
         - --path.rootfs=/host/root
+        - --path.procfs=/host/root/proc
         - --path.udev.data=/host/root/run/udev/data
         - --no-collector.wifi
         - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)
@@ -86,7 +87,7 @@ spec:
       - args:
         - --secure-listen-address=[$(IP)]:9100
         - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
-        - --upstream=http://127.0.0.1:9100/
+        - --upstream=http://127.0.0.1:9101/
         - --tls-cert-file=/etc/tls/private/tls.crt
         - --tls-private-key-file=/etc/tls/private/tls.key
         - --client-ca-file=/etc/tls/client/client-ca.crt

assets/prometheus-k8s/prometheus.yaml

@@ -223,6 +223,7 @@ spec:
     runAsNonRoot: true
     runAsUser: 65534
   serviceAccountName: prometheus-k8s
+  serviceDiscoveryRole: EndpointSlice
   serviceMonitorNamespaceSelector:
     matchLabels:
       openshift.io/cluster-monitoring: "true"

assets/prometheus-k8s/role-specific-namespaces.yaml

@@ -12,11 +12,18 @@ items:
     name: prometheus-k8s
     namespace: default
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get
@@ -50,11 +57,18 @@ items:
     name: prometheus-k8s
     namespace: kube-system
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get
@@ -88,11 +102,18 @@ items:
     name: prometheus-k8s
     namespace: openshift-monitoring
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get
@@ -126,11 +147,18 @@ items:
     name: prometheus-k8s
     namespace: openshift-user-workload-monitoring
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get

assets/prometheus-operator-user-workload/cluster-role.yaml

@@ -23,6 +23,7 @@ rules:
   - thanosrulers/finalizers
   - thanosrulers/status
   - servicemonitors
+  - servicemonitors/status
   - podmonitors
   - probes
   - prometheusrules
@@ -95,12 +96,13 @@ rules:
   verbs:
   - get
 - apiGroups:
-  - ""
+  - discovery.k8s.io
   resources:
-  - endpoints
+  - endpointslices
   verbs:
   - get
   - create
+  - list
   - update
   - delete
 - apiGroups:

assets/prometheus-operator-user-workload/deployment.yaml

@@ -34,7 +34,7 @@ spec:
       - args:
         - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.81.0
         - --kubelet-endpoints=true
-        - --kubelet-endpointslice=false
+        - --kubelet-endpointslice=true
         - --prometheus-instance-namespaces=openshift-user-workload-monitoring
         - --alertmanager-instance-namespaces=openshift-user-workload-monitoring
         - --thanos-ruler-instance-namespaces=openshift-user-workload-monitoring

assets/prometheus-operator/cluster-role.yaml

@@ -23,6 +23,7 @@ rules:
   - thanosrulers/finalizers
   - thanosrulers/status
   - servicemonitors
+  - servicemonitors/status
   - podmonitors
   - probes
   - prometheusrules
@@ -95,12 +96,13 @@ rules:
   verbs:
   - get
 - apiGroups:
-  - ""
+  - discovery.k8s.io
   resources:
-  - endpoints
+  - endpointslices
   verbs:
   - get
   - create
+  - list
   - update
   - delete
 - apiGroups:

assets/prometheus-operator/deployment.yaml

@@ -35,7 +35,7 @@ spec:
         - --kubelet-service=kube-system/kubelet
         - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.81.0
         - --kubelet-endpoints=true
-        - --kubelet-endpointslice=false
+        - --kubelet-endpointslice=true
         - --prometheus-instance-namespaces=openshift-monitoring
         - --thanos-ruler-instance-namespaces=openshift-monitoring
         - --alertmanager-instance-namespaces=openshift-monitoring

assets/prometheus-user-workload/prometheus.yaml

@@ -278,6 +278,7 @@ spec:
     seccompProfile:
       type: RuntimeDefault
   serviceAccountName: prometheus-user-workload
+  serviceDiscoveryRole: EndpointSlice
   serviceMonitorNamespaceSelector:
     matchExpressions:
     - key: openshift.io/cluster-monitoring

assets/prometheus-user-workload/role-specific-namespaces.yaml

@@ -12,11 +12,18 @@ items:
     name: prometheus-user-workload
     namespace: openshift-user-workload-monitoring
   rules:
+  - apiGroups:
+    - discovery.k8s.io
+    resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
   - apiGroups:
     - ""
     resources:
     - services
-    - endpoints
     - pods
     verbs:
     - get

hack/build-jsonnet.sh

@@ -37,7 +37,7 @@ for file in "${files[@]}"; do
 	}&
 
 	# wait for at least one of the jobs to finish if there are more than maxProc jobs
-	while [[ $(jobs -r | wc -l ) -ge "$maxProc" ]]; do wait -n; done
+	while [[ $(jobs -r | wc -l ) -ge "$maxProc" ]]; do wait ; done
 done
 # wait for all jobs to finish
 wait

jsonnet/components/admission-webhook.libsonnet

@@ -1,6 +1,6 @@
 local tlsVolumeName = 'prometheus-operator-admission-webhook-tls';
 local admissionWebhook = import 'github.com/prometheus-operator/prometheus-operator/jsonnet/prometheus-operator/admission-webhook.libsonnet';
-local antiAffinity = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/addons/anti-affinity.libsonnet';
+local antiAffinity = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/addons/anti-affinity.libsonnet';
 local withDescription = (import '../utils/add-annotations.libsonnet').withDescription;
 
 function(params)

jsonnet/components/alertmanager-user-workload.libsonnet

@@ -1,6 +1,6 @@
-local alertmanager = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/alertmanager.libsonnet';
+local alertmanager = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/alertmanager.libsonnet';
 // TODO: replace current addition of kube-rbac-proxy with upstream lib
-// local krp = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet';
+// local krp = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet';
 local generateCertInjection = import '../utils/generate-certificate-injection.libsonnet';
 local generateSecret = import '../utils/generate-secret.libsonnet';
 local withDescription = (import '../utils/add-annotations.libsonnet').withDescription;

jsonnet/components/alertmanager.libsonnet

@@ -1,6 +1,6 @@
-local alertmanager = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/alertmanager.libsonnet';
+local alertmanager = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/alertmanager.libsonnet';
 // TODO: replace current addition of kube-rbac-proxy with upstream lib
-// local krp = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet';
+// local krp = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet';
 local generateCertInjection = import '../utils/generate-certificate-injection.libsonnet';
 local generateSecret = import '../utils/generate-secret.libsonnet';
 local withDescription = (import '../utils/add-annotations.libsonnet').withDescription;

jsonnet/components/cluster-monitoring-operator.libsonnet

@@ -1,7 +1,7 @@
 local metrics = import 'github.com/openshift/telemeter/jsonnet/telemeter/metrics.jsonnet';
 
 local cmoRules = import './../rules.libsonnet';
-local kubePrometheus = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/mixin/custom.libsonnet';
+local kubePrometheus = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/mixin/custom.libsonnet';
 
 local defaults = {
   local defaults = self,

jsonnet/components/control-plane.libsonnet

@@ -1,5 +1,5 @@
 local generateServiceMonitor = import '../utils/generate-service-monitors.libsonnet';
-local controlPlane = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet';
+local controlPlane = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet';
 
 function(params)
   local cfg = params;

jsonnet/components/kube-state-metrics.libsonnet

@@ -2,7 +2,7 @@ local tmpVolumeName = 'volume-directive-shadow';
 local tlsVolumeName = 'kube-state-metrics-tls';
 local crsVolumeName = 'kube-state-metrics-custom-resource-state-configmap';
 
-local kubeStateMetrics = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet';
+local kubeStateMetrics = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet';
 local kubeStateMetricsCRS = import '../utils/kube-state-metrics-custom-resource-state.libsonnet';
 local generateSecret = import '../utils/generate-secret.libsonnet';
 local generateServiceMonitor = import '../utils/generate-service-monitors.libsonnet';

jsonnet/components/node-exporter.libsonnet

@@ -90,7 +90,7 @@ local acceleratorsConfigData = [
 
 ];
 
-local nodeExporter = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/node-exporter.libsonnet';
+local nodeExporter = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/node-exporter.libsonnet';
 local generateSecret = import '../utils/generate-secret.libsonnet';
 local generateServiceMonitor = import '../utils/generate-service-monitors.libsonnet';
 local withDescription = (import '../utils/add-annotations.libsonnet').withDescription;

jsonnet/components/prometheus-operator-user-workload.libsonnet

@@ -1,6 +1,6 @@
 local tlsVolumeName = 'prometheus-operator-user-workload-tls';
 
-local operator = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet';
+local operator = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet';
 local generateSecret = import '../utils/generate-secret.libsonnet';
 local rbac = import '../utils/rbac.libsonnet';
 local withDescription = (import '../utils/add-annotations.libsonnet').withDescription;

jsonnet/components/prometheus-operator.libsonnet

@@ -1,7 +1,7 @@
 local tlsVolumeName = 'prometheus-operator-tls';
 
 local generateCertInjection = import '../utils/generate-certificate-injection.libsonnet';
-local operator = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet';
+local operator = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet';
 local conversionWebhook = import 'github.com/prometheus-operator/prometheus-operator/jsonnet/prometheus-operator/conversion.libsonnet';
 local generateSecret = import '../utils/generate-secret.libsonnet';
 local rbac = import '../utils/rbac.libsonnet';

jsonnet/components/prometheus-user-workload.libsonnet

@@ -3,7 +3,7 @@ local generateSecret = import '../utils/generate-secret.libsonnet';
 local withDescription = (import '../utils/add-annotations.libsonnet').withDescription;
 local requiredClusterRoles = (import '../utils/add-annotations.libsonnet').requiredClusterRoles;
 
-local prometheus = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/prometheus.libsonnet';
+local prometheus = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/prometheus.libsonnet';
 
 function(params)
   local cfg = params;

jsonnet/components/prometheus.libsonnet

@@ -2,7 +2,7 @@ local metrics = import 'github.com/openshift/telemeter/jsonnet/telemeter/metrics
 
 local generateCertInjection = import '../utils/generate-certificate-injection.libsonnet';
 local generateSecret = import '../utils/generate-secret.libsonnet';
-local prometheus = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/components/prometheus.libsonnet';
+local prometheus = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/components/prometheus.libsonnet';
 local withDescription = (import '../utils/add-annotations.libsonnet').withDescription;
 local requiredClusterRoles = (import '../utils/add-annotations.libsonnet').requiredClusterRoles;
 

jsonnet/components/telemeter-client.libsonnet

@@ -8,7 +8,7 @@ function(params) {
     _config+:: {
       namespace: cfg.namespace,
       tlsCipherSuites: [
-        // List from https://github.com/prometheus-operator/kube-prometheus/blob/master/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet
+        // List from https://github.com/slashpai/kube-prometheus/blob/master/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet
         'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256',
         'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256',
         'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384',

jsonnet/jsonnetfile.json

@@ -4,11 +4,11 @@
     {
       "source": {
         "git": {
-          "remote": "https://github.com/prometheus-operator/kube-prometheus",
+          "remote": "https://github.com/slashpai/kube-prometheus",
           "subdir": "jsonnet/kube-prometheus"
         }
       },
-      "version": "main"
+      "version": "endpoint-slice"
     },
     {
       "source": {

jsonnet/jsonnetfile.lock.json

@@ -174,16 +174,6 @@
       "sum": "53k2pqb4mjABCulTT/H5d7pQDiemIOO23gzfon7g9K4=",
       "name": "telemeter-client"
     },
-    {
-      "source": {
-        "git": {
-          "remote": "https://github.com/prometheus-operator/kube-prometheus.git",
-          "subdir": "jsonnet/kube-prometheus"
-        }
-      },
-      "version": "7e5a571a3fb735c78e17c76a637eb7e8bb5dd086",
-      "sum": "uTw/Mj+X91S+oqUpAX81xcfWPDlox0tdSZY/YBw7nGE="
-    },
     {
       "source": {
         "git": {
@@ -248,6 +238,26 @@
       "sum": "cxAPQovFkM16zNB5/94O+sk/n3SETk6ao6Oas2Sa6RE=",
       "name": "pyrra"
     },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/slashpai/kube-prometheus.git",
+          "subdir": "jsonnet/kube-prometheus"
+        }
+      },
+      "version": "a8542720ec22d0147fe6e7dacaddfa576d14ccae",
+      "sum": "1M76lVs3dZGYl52Wy3vg59zyL/qGcii3E+JYsL+kF0M="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/slashpai/prometheus-operator.git",
+          "subdir": "jsonnet/prometheus-operator"
+        }
+      },
+      "version": "f8960863958daef20c77e7be24d99e0d6c2aa6b6",
+      "sum": "Afa/g9EOG+6CGLMuMHLh9IdaWQHDWXbREOHNT9ndJ2w="
+    },
     {
       "source": {
         "git": {

jsonnet/main.jsonnet

@@ -432,7 +432,7 @@ local inCluster =
     openshiftStateMetrics: openshiftStateMetrics($.values.openshiftStateMetrics),
   } +
   (import './utils/anti-affinity.libsonnet') +
-  (import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/addons/ksm-lite.libsonnet') +
+  (import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/addons/ksm-lite.libsonnet') +
   (import './utils/ibm-cloud-managed-profile.libsonnet') +
   (import './components/metrics-server-audit.libsonnet') +
   {};  // Including empty object to simplify adding and removing imports during development

jsonnet/utils/anti-affinity.libsonnet

@@ -1,4 +1,4 @@
-local addon = import 'github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus/addons/anti-affinity.libsonnet';
+local addon = import 'github.com/slashpai/kube-prometheus/jsonnet/kube-prometheus/addons/anti-affinity.libsonnet';
 
 addon {
   values+:: {