Skip to content

Commit b7e65a7

Browse files
dtfranzdtfranz
committed
Prometheus Metrics
Adds prometheus to the test-e2e Makefile target, which stands up a barebones prometheus scraper to gather metrics from the operator-controller and catalogd pods during the e2e test run. When finished, the prometheus server is queried for a raw output of the metrics and stores it in metrics.out. These metrics will be analyzed in a later PR. Signed-off-by: Daniel Franz <[email protected]>
1 parent 44de6f2 commit b7e65a7

File tree

3 files changed

+238
-1
lines changed

3 files changed

+238
-1
lines changed

‎Makefile

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,17 @@ image-registry: ## Build the testdata catalog used for e2e tests and push it to
256256
test-e2e: KIND_CLUSTER_NAME := operator-controller-e2e
257257
test-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/e2e
258258
test-e2e: GO_BUILD_EXTRA_FLAGS := -cover
259-
test-e2e: run image-registry e2e e2e-coverage kind-clean #HELP Run e2e test suite on local kind cluster
259+
test-e2e: run image-registry prometheus e2e e2e-coverage e2e-metrics kind-clean #HELP Run e2e test suite on local kind cluster
260+
261+
.PHONY: prometheus
262+
prometheus: PROMETHEUS_NAMESPACE := olmv1-system
263+
prometheus: PROMETHEUS_VERSION := v0.83.0
264+
prometheus: #HELP Deploy Prometheus into specified namespace
265+
./hack/test/setup-monitoring.sh $(PROMETHEUS_NAMESPACE) $(PROMETHEUS_VERSION) $(KUSTOMIZE)
266+
267+
.PHONY: e2e-metrics
268+
e2e-metrics: #HELP Request metrics from prometheus; place in ARTIFACT_PATH if set
269+
curl http://localhost:30900/metrics > $(if $(ARTIFACT_PATH),$(ARTIFACT_PATH),.)/metrics.out
260270

261271
.PHONY: extension-developer-e2e
262272
extension-developer-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager

‎hack/test/setup-monitoring.sh

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
help="setup-monitoring.sh is used to set up prometheus monitoring for e2e testing.
6+
7+
Usage:
8+
setup-monitoring.sh [PROMETHEUS_NAMESPACE] [PROMETHEUS_VERSION] [KUSTOMIZE]
9+
"
10+
11+
if [[ "$#" -ne 3 ]]; then
12+
echo "Illegal number of arguments passed"
13+
echo "${help}"
14+
exit 1
15+
fi
16+
17+
NAMESPACE=$1
18+
PROMETHEUS_VERSION=$2
19+
KUSTOMIZE=$3
20+
21+
TMPDIR=$(mktemp -d)
22+
trap 'echo "Cleaning up ${TMPDIR}"; rm -rf "${TMPDIR}"' EXIT
23+
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/${PROMETHEUS_VERSION}/kustomization.yaml" > "${TMPDIR}/kustomization.yaml"
24+
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/${PROMETHEUS_VERSION}/bundle.yaml" > "${TMPDIR}/bundle.yaml"
25+
(cd ${TMPDIR} && ${KUSTOMIZE} edit set namespace ${NAMESPACE}) && kubectl create -k "${TMPDIR}"
26+
kubectl wait --for=condition=Ready pods -n ${NAMESPACE} -l app.kubernetes.io/name=prometheus-operator
27+
28+
kubectl apply -f - << EOF
29+
apiVersion: v1
30+
kind: ServiceAccount
31+
metadata:
32+
name: prometheus
33+
namespace: ${NAMESPACE}
34+
---
35+
apiVersion: rbac.authorization.k8s.io/v1
36+
kind: ClusterRole
37+
metadata:
38+
name: prometheus
39+
rules:
40+
- apiGroups: [""]
41+
resources:
42+
- nodes
43+
- nodes/metrics
44+
- services
45+
- endpoints
46+
- pods
47+
verbs: ["get", "list", "watch"]
48+
- apiGroups: [""]
49+
resources:
50+
- configmaps
51+
verbs: ["get"]
52+
- apiGroups:
53+
- discovery.k8s.io
54+
resources:
55+
- endpointslices
56+
verbs: ["get", "list", "watch"]
57+
- apiGroups:
58+
- networking.k8s.io
59+
resources:
60+
- ingresses
61+
verbs: ["get", "list", "watch"]
62+
- nonResourceURLs: ["/metrics"]
63+
verbs: ["get"]
64+
---
65+
apiVersion: rbac.authorization.k8s.io/v1
66+
kind: ClusterRoleBinding
67+
metadata:
68+
name: prometheus
69+
roleRef:
70+
apiGroup: rbac.authorization.k8s.io
71+
kind: ClusterRole
72+
name: prometheus
73+
subjects:
74+
- kind: ServiceAccount
75+
name: prometheus
76+
namespace: ${NAMESPACE}
77+
EOF
78+
79+
kubectl apply -f - << EOF
80+
apiVersion: monitoring.coreos.com/v1
81+
kind: Prometheus
82+
metadata:
83+
name: prometheus
84+
namespace: ${NAMESPACE}
85+
spec:
86+
logLevel: debug
87+
serviceAccountName: prometheus
88+
scrapeTimeout: 30s
89+
scrapeInterval: 1m
90+
securityContext:
91+
runAsNonRoot: true
92+
runAsUser: 65534
93+
seccompProfile:
94+
type: RuntimeDefault
95+
serviceMonitorSelector: {}
96+
EOF
97+
98+
kubectl apply -f - << EOF
99+
apiVersion: networking.k8s.io/v1
100+
kind: NetworkPolicy
101+
metadata:
102+
name: prometheus
103+
namespace: ${NAMESPACE}
104+
spec:
105+
podSelector:
106+
matchLabels:
107+
app.kubernetes.io/name: prometheus
108+
policyTypes:
109+
- Egress
110+
- Ingress
111+
egress:
112+
- {} # Allows all egress traffic for metrics requests
113+
ingress:
114+
- {} # Allows us to query prometheus
115+
EOF
116+
117+
# Give the operator time to create the pod
118+
kubectl wait --for=create pods -n ${NAMESPACE} prometheus-prometheus-0 --timeout=60s
119+
kubectl wait --for=condition=Ready pods -n ${NAMESPACE} prometheus-prometheus-0 --timeout=120s
120+
121+
# Authentication token for the scrape requests
122+
kubectl apply -f - <<EOF
123+
apiVersion: v1
124+
kind: Secret
125+
type: kubernetes.io/service-account-token
126+
metadata:
127+
name: prometheus-metrics-token
128+
namespace: ${NAMESPACE}
129+
annotations:
130+
kubernetes.io/service-account.name: prometheus
131+
EOF
132+
133+
# ServiceMonitors for operator-controller and catalogd
134+
kubectl apply -f - <<EOF
135+
apiVersion: monitoring.coreos.com/v1
136+
kind: ServiceMonitor
137+
metadata:
138+
name: operator-controller-controller-manager-metrics-monitor
139+
namespace: ${NAMESPACE}
140+
spec:
141+
endpoints:
142+
- path: /metrics
143+
port: https
144+
scheme: https
145+
authorization:
146+
credentials:
147+
name: prometheus-metrics-token
148+
key: token
149+
tlsConfig:
150+
serverName: operator-controller-service.${NAMESPACE}.svc
151+
insecureSkipVerify: false
152+
ca:
153+
secret:
154+
name: olmv1-cert
155+
key: ca.crt
156+
cert:
157+
secret:
158+
name: olmv1-cert
159+
key: tls.crt
160+
keySecret:
161+
name: olmv1-cert
162+
key: tls.key
163+
selector:
164+
matchLabels:
165+
control-plane: operator-controller-controller-manager
166+
EOF
167+
168+
CATD_SECRET=$(kubectl get secret -n ${NAMESPACE} -o jsonpath="{.items[*].metadata.name}" | tr ' ' '\n' | grep '^catalogd-service-cert')
169+
170+
kubectl apply -f - <<EOF
171+
apiVersion: monitoring.coreos.com/v1
172+
kind: ServiceMonitor
173+
metadata:
174+
name: catalogd-controller-manager-metrics-monitor
175+
namespace: ${NAMESPACE}
176+
spec:
177+
endpoints:
178+
- path: /metrics
179+
port: metrics
180+
scheme: https
181+
authorization:
182+
credentials:
183+
name: prometheus-metrics-token
184+
key: token
185+
tlsConfig:
186+
serverName: catalogd-service.${NAMESPACE}.svc
187+
insecureSkipVerify: false
188+
ca:
189+
secret:
190+
name: ${CATD_SECRET}
191+
key: ca.crt
192+
cert:
193+
secret:
194+
name: ${CATD_SECRET}
195+
key: tls.crt
196+
keySecret:
197+
name: ${CATD_SECRET}
198+
key: tls.key
199+
selector:
200+
matchLabels:
201+
app.kubernetes.io/name: catalogd
202+
EOF
203+
204+
# NodePort service to allow querying prometheus from outside the cluster
205+
# NOTE: This NodePort must also be configured in kind-config.yaml
206+
kubectl apply -f - <<EOF
207+
apiVersion: v1
208+
kind: Service
209+
metadata:
210+
name: prometheus-service
211+
namespace: ${NAMESPACE}
212+
spec:
213+
type: NodePort
214+
ports:
215+
- name: web
216+
nodePort: 30900
217+
port: 9090
218+
protocol: TCP
219+
targetPort: web
220+
selector:
221+
prometheus: prometheus
222+
EOF

‎kind-config.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ nodes:
88
hostPort: 30000
99
listenAddress: "127.0.0.1"
1010
protocol: tcp
11+
# prometheus metrics service's NodePort
12+
- containerPort: 30900
13+
hostPort: 30900
14+
listenAddress: "127.0.0.1"
15+
protocol: tcp
1116
kubeadmConfigPatches:
1217
- |
1318
kind: ClusterConfiguration

0 commit comments

Comments
 (0)