Add Prometheus instance to development setup with Tilt

.gitignore

@@ -26,6 +26,9 @@ dist/*
 # Helm dependency chart archives
 charts/**/*.tgz
 
+# Tilt cache
+.tiltcache/
+
 # editor and IDE paraphernalia
 .idea
 .vscode

Makefile

@@ -87,7 +87,7 @@ setup-test-e2e: kind ## Set up a Kind cluster for e2e tests if it does not exist
 	esac
 
 # E2E test dependency versions
-E2E_PROMETHEUS_OPERATOR_VERSION ?= v0.82.2
+E2E_PROMETHEUS_OPERATOR_VERSION ?= v0.92.1
 E2E_CERTMANAGER_VERSION ?= v1.21.0
 
 .PHONY: test-e2e

Tiltfile

@@ -17,6 +17,23 @@ is_kind = cluster_name.startswith('kind-')
 load('ext://cert_manager', 'deploy_cert_manager')
 deploy_cert_manager(version='v1.21.0', load_to_kind=is_kind, kind_cluster_name=cluster_name.removeprefix('kind-'))
 
+def deploy_prometheus_operator(version='v0.92.1'):
+    local('kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -', quiet=True, echo_off=True)
+
+    print('Installing prometheus-operator')
+    cache_file = '.tiltcache/prometheus-operator-{}.yaml'.format(version)
+    if not os.path.exists(cache_file):
+        local("mkdir -p .tiltcache && curl -sL https://github.com/prometheus-operator/prometheus-operator/releases/download/{}/bundle.yaml | sed 's/namespace: default/namespace: monitoring/g' > {}".format(version, cache_file), quiet=True, echo_off=True)
+    local('kubectl apply --server-side -n monitoring -f {}'.format(cache_file), quiet=True, echo_off=True)
+
+    print('Waiting for prometheus-operator to start')
+    local('kubectl wait --for=condition=Available --timeout=300s -n monitoring deployment/prometheus-operator', quiet=True, echo_off=True)
+
+    k8s_yaml('./config/develop/prometheus.yaml')
+    k8s_resource(new_name = 'prometheus', objects = ['prometheus:prometheus'], port_forwards = '9090', extra_pod_selectors = [{'app.kubernetes.io/name': 'prometheus'}], labels = ['observability'])
+
+deploy_prometheus_operator()
+
 docker_build('controller:latest', '.', only=[
     'api/', 'cmd/', 'internal/', 'go.mod', 'go.sum'
 ])

config/develop/kustomization.yaml

@@ -1,8 +1,15 @@
 resources:
 - ../default
+- ../prometheus
 
 patches:
  - path: manager_patch.yaml
    target:
      kind: Deployment
      name: controller-manager
+ - target:
+     kind: ServiceMonitor
+   patch: |
+     - op: replace
+       path: /metadata/namespace
+       value: network-operator-system

config/develop/manager_patch.yaml

@@ -3,6 +3,7 @@
   value:
     - --leader-elect=false
     - --health-probe-bind-address=:8081
+    - --metrics-bind-address=:8443
     - --provider=openconfig
     - --requeue-interval=30s
     - --max-concurrent-reconciles=5

config/develop/prometheus.yaml

@@ -0,0 +1,58 @@
+# Prometheus instance for local development.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+---
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: prometheus
+  namespace: monitoring
+spec:
+  replicas: 1
+  serviceAccountName: prometheus
+  serviceMonitorNamespaceSelector: {}
+  serviceMonitorSelector: {}
+  podMonitorNamespaceSelector: {}
+  podMonitorSelector: {}
+  resources:
+    requests:
+      memory: 400Mi
+---
+# RBAC setup based on https://prometheus-operator.dev/docs/platform/rbac/
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "nodes/metrics", "services", "endpoints", "pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses"]
+    verbs: ["get", "list", "watch"]
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+  - kind: ServiceAccount
+    name: prometheus
+    namespace: monitoring