feat(kube-monitoring): Add Pod service discovery (#553)

* feat(kube-monitoring): Add Pod service discovery

* chore(kube-monitoring): adapt charts tests for pods sd

* docs(kube-monitoring): add important note to Pod SD

Co-authored-by: Tommy Sauer <tommy.sauer@sap.com>

* chore(kube-monitoring): more precise selector description

Co-authored-by: Tommy Sauer <tommy.sauer@sap.com>

---------

Co-authored-by: Tommy Sauer <tommy.sauer@sap.com>

Author: richardtief

kube-monitoring/README.md

@@ -124,6 +124,12 @@ Greenhouse regularly performs integration tests that are bundled with **kube-mon
 | `alerts.alertmanager.tlsConfig.cert`          | TLS certificate for communication with Alertmanager                                                                 | `Secret`                 |
 | `alerts.alertmanager.tlsConfig.key`           | TLS key for communication with Alertmanager                                                                         | `Secret`                 |
 
+## Service Discovery
+
+The **kube-monitoring** Plugin uses a PodMonitor to automatically discover the Prometheus metrics of the Kubernetes Pods in any Namespace. The PodMonitor is configured to detect the metrics endpoint of the pods with the port name `metrics` **and** the label `greenhouse/scrape: “true”`.
+
+*Important*: The label needs to be added manually to have the pod scraped and the port name needs to match.
+
 ## Examples
 
 ### Deploy kube-monitoring into a remote cluster

kube-monitoring/charts/Chart.yaml

@@ -8,7 +8,7 @@ maintainers:
 name: kube-monitoring
 sources:
   - https://github.com/cloudoperators/greenhouse-extensions
-version: 0.19.2
+version: 0.20.0
 # prometheus-operator app version
 appVersion: v0.77.1
 keywords:

kube-monitoring/charts/templates/pmon.yaml

@@ -0,0 +1,27 @@
+{{- if and .Values.kubeMonitoring.prometheus.enabled .Values.kubeMonitoring.serviceDiscovery.pods.enabled }}
+{{- $values := .Values.kubeMonitoring.serviceDiscovery.pods }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ $.Release.Name }}-pod-sd
+  labels:
+    {{- include "kube-prometheus-stack.labels" . | nindent 4 }}
+spec:
+  {{- include "servicemonitor.scrapeLimits" . | nindent 2 }}
+  podMetricsEndpoints:
+{{ tpl (toYaml $values.podMetricsEndpoints) . | indent 4 }}
+  {{- if $values.jobLabel }}
+  jobLabel: {{ $values.pods.jobLabel }}
+  {{- end }}
+  namespaceSelector:
+{{ toYaml $values.namespaceSelector | indent 4 }}
+  selector:
+{{ toYaml $values.selector | indent 4 }}
+  {{- if $values.podTargetLabels }}
+  podTargetLabels:
+{{ toYaml $values.podTargetLabels | indent 4 }}
+  {{- end }}
+  {{- if $values.sampleLimit }}
+  sampleLimit: {{ $values.sampleLimit }}
+  {{- end }}
+{{- end }}

kube-monitoring/charts/templates/podmonitor-sd.yaml

@@ -54,9 +54,12 @@ data:
         verify "there is 1 statefulset named 'prometheus-{{ .Release.Name }}'"
     }
 
-    @test "Verify creation of the {{ .Release.Name }}-sd Podmonitor" {
-        verify "there is 1 podmonitor named '{{ .Release.Name }}-sd'"
+    
+    {{- if and .Values.kubeMonitoring.prometheus.enabled .Values.kubeMonitoring.serviceDiscovery.pods.enabled }}
+    @test "Verify creation of the {{ .Release.Name }}-pod-sd Podmonitor" {
+        verify "there is 1 podmonitor named '{{ .Release.Name }}-pod-sd'"
     }
+    {{- end }}
 
     @test "Verify creation of required custom resource definitions (CRDs) for {{ .Release.Name }}" {
         verify "there is 1 customresourcedefinition named 'prometheuses'"

kube-monitoring/charts/templates/tests/test-kube-monitoring-config.yaml

@@ -197,6 +197,70 @@ kubeMonitoring:
       - services=[*]
       - statefulsets=[*]
 
+  ## Service discovery configuration
+  ##
+  serviceDiscovery:
+
+    ## Configuration for the Prometheus Operator to discover pods
+    ##
+    pods:
+      enabled: true
+
+      ## Endpoints of the selected pods to be monitored
+      ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmetricsendpoint
+      ##
+      podMetricsEndpoints:
+          ## Monitor Pods with the following port name
+        - port: metrics
+
+          ## RelabelConfigs to apply to samples before scraping
+          ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig
+          ##
+          relabelings: []
+          # - sourceLabels: [__meta_kubernetes_pod_node_name]
+          #   separator: ;
+          #   regex: ^(.*)$
+          #   targetLabel: nodename
+          #   replacement: $1
+          #   action: replace
+
+          ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion.
+          ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig
+          ##
+          metricRelabelings: []
+          # - action: keep
+          #   regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+'
+          #   sourceLabels: [__name__]
+
+      ## Pod label for use in assembling a job name of the form <label value>-<port>
+      ## If no label is specified, the pod endpoint name is used.
+      ##
+      jobLabel: ""
+
+      ## Namespaces from which pods are selected
+      ##
+      namespaceSelector:
+        ## Match any namespace
+        ##
+        any: true
+
+        ## Explicit list of namespace names to select
+        ##
+        # matchNames: []
+
+      ## Custom definition to pick up specific pods with an expression.
+      selector:
+        matchExpressions: []
+
+        ## label selector for pods
+        ##
+        matchLabels:
+          greenhouse/scrape: "true"
+
+      ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+      ##
+      # sampleLimit: 0
+
   ## Configuration for the Prometheus instance
   ##
   prometheus:

kube-monitoring/charts/values.yaml

@@ -6,15 +6,15 @@ kind: PluginDefinition
 metadata:
   name: kube-monitoring
 spec:
-  version: 2.6.5
+  version: 2.7.0
   displayName: Kubernetes monitoring
   description: Native deployment and management of Prometheus along with Kubernetes cluster monitoring components.
   docMarkDownUrl: https://raw.githubusercontent.com/cloudoperators/greenhouse-extensions/main/kube-monitoring/README.md
   icon: https://raw.githubusercontent.com/cloudoperators/greenhouse-extensions/main/kube-monitoring/logo.png
   helmChart:
     name: kube-monitoring
     repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts
-    version: 0.19.2
+    version: 0.20.0
   options:
     - name: global.commonLabels
       description: Labels to add to all resources. This can be used to add a support group or service to all alerts.