Terraform setup: enable Kubernetes Monitoring

Author: ppcano

terraform/README.md

@@ -82,3 +82,19 @@ To use a locally built QuickPizza image instead of the default from the registry
    ```bash
    terraform apply
    ```
+
+## Enable Kubernetes Monitoring
+
+Update `terraform.tfvars` and set the required settings:
+
+```hcl
+enable_k8s_monitoring = true
+cluster_name = ""
+externalservices_prometheus_host = ""
+externalservices_prometheus_basicauth_username = ""
+externalservices_prometheus_basicauth_password = ""
+
+externalservices_loki_host = ""
+externalservices_loki_basicauth_username = ""
+externalservices_loki_basicauth_password = ""
+```

terraform/alloy.tf

@@ -88,6 +88,10 @@ resource "kubernetes_deployment" "alloy" {
               name = kubernetes_secret.alloy_credentials.metadata[0].name
             }
           }
+          env {
+            name  = "KUBERNETES_CLUSTER_NAME"
+            value = var.cluster_name
+          }
           env {
             name  = "QUICKPIZZA_PYROSCOPE_SERVICE_GIT_REF"
             value = var.quickpizza_git_ref

terraform/alloy/config.alloy

@@ -76,10 +76,26 @@ prometheus.scrape "application_pods" {
   targets = discovery.relabel.application_pods.output
 }
 
+discovery.relabel "application_pods_logs" {
+  // Enable logs in Kubernetes Monitoring
+  // Only add cluster_name label if KUBERNETES_CLUSTER_NAME env var is set
+  rule {
+    target_label = "cluster"
+    replacement = env("KUBERNETES_CLUSTER_NAME")
+  }
+  rule {
+    target_label = "pod"
+    source_labels = [
+      "__meta_kubernetes_pod_name",
+    ]
+  }
+  targets = discovery.relabel.application_pods.output
+}
+
 // Logs: application pods
 loki.source.kubernetes "application_pods" {
   forward_to = [grafana_cloud.stack.receivers.logs]
-  targets = discovery.relabel.application_pods.output
+  targets = discovery.relabel.application_pods_logs.output
 }
 
 discovery.relabel "application_pods_profiles" {

terraform/k8-monitoring.tf

@@ -0,0 +1,64 @@
+resource "helm_release" "grafana-k8s-monitoring" {
+  count = var.enable_k8s_monitoring ? 1 : 0
+  name             = "grafana-k8s-monitoring"
+  repository       = "https://grafana.github.io/helm-charts"
+  chart            = "k8s-monitoring"
+  version          = "^1"
+  namespace        = "quickpizza-monitoring"
+  create_namespace = true
+  atomic           = true
+  timeout          = 300
+
+  lifecycle {
+    # this causes a warning saying it's redundant, but it is intentional,
+    # see https://github.com/hashicorp/terraform-provider-helm/issues/1315
+    ignore_changes = [metadata]
+  }
+
+  values = [file("${path.module}/k8-monitoring.yaml")]
+
+  set {
+    name  = "cluster.name"
+    value = var.cluster_name
+  }
+
+  set {
+    name  = "externalServices.prometheus.host"
+    value = var.externalservices_prometheus_host
+  }
+
+  set_sensitive {
+    name  = "externalServices.prometheus.basicAuth.username"
+    value = var.externalservices_prometheus_basicauth_username
+  }
+
+  set_sensitive {
+    name  = "externalServices.prometheus.basicAuth.password"
+    value = var.externalservices_prometheus_basicauth_password
+  }
+
+  set {
+    name  = "externalServices.loki.host"
+    value = var.externalservices_loki_host
+  }
+
+  set_sensitive {
+    name  = "externalServices.loki.basicAuth.username"
+    value = var.externalservices_loki_basicauth_username
+  }
+
+  set_sensitive {
+    name  = "externalServices.loki.basicAuth.password"
+    value = var.externalservices_loki_basicauth_password
+  }
+
+  set {
+    name  = "opencost.opencost.exporter.defaultClusterId"
+    value = var.cluster_name
+  }
+
+  set {
+    name  = "opencost.opencost.prometheus.external.url"
+    value = format("%s/api/prom", var.externalservices_prometheus_host)
+  }
+}

terraform/k8-monitoring.yaml

@@ -0,0 +1,25 @@
+metrics:
+  enabled: true
+  kepler:
+    enabled: true
+
+alloy-logs:
+  alloy:
+    mounts:
+      dockercontainers: true
+
+logs:
+  enabled: true
+  pod_logs:
+    enabled: true
+    # The Alloy pod in quickpizza ns collects quickpizza logs
+    excludeNamespaces: [quickpizza]
+
+traces:
+  enabled: false
+
+profiles:
+  enabled: false
+
+kepler:
+  enabled: true

terraform/terraform.tf

@@ -4,6 +4,10 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = ">= 2.0.1"
     }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.17.0"
+    }
   }
   required_version = ">= 1.8.0"
 }
@@ -12,3 +16,10 @@ provider "kubernetes" {
   config_path    = "~/.kube/config"
   config_context = "minikube"
 }
+
+provider "helm" {
+  kubernetes {
+    config_path    = "~/.kube/config"
+    config_context = "minikube"
+  }
+}

terraform/terraform.tfvars.local

@@ -12,4 +12,15 @@ grafana_cloud_token = "glc_your_token_here"
 # Optional: Faro configuration for Frontend Observability
 # Get these from your Grafana Cloud Faro app configuration
 # quickpizza_conf_faro_url = ""
-# quickpizza_conf_faro_app_name = ""
+# quickpizza_conf_faro_app_name = ""
+
+
+# enable_k8s_monitoring = true
+# cluster_name = ""
+# externalservices_prometheus_host = ""
+# externalservices_prometheus_basicauth_username = ""
+# externalservices_prometheus_basicauth_password = ""
+
+# externalservices_loki_host = ""
+# externalservices_loki_basicauth_username = ""
+# externalservices_loki_basicauth_password = ""

terraform/variables.tf

@@ -65,4 +65,50 @@ variable "quickpizza_log_level" {
   description = "The Log Level to use for the QuickPizza Demo Application, for example \"info\" or \"debug\"."
   nullable    = false
   type        = string
+}
+
+variable "enable_k8s_monitoring" {
+  description = "Enable or disable Kubernetes monitoring Helm chart"
+  type        = bool
+  default     = false
+}
+
+variable "cluster_name" {
+  type    = string
+  default     = null
+}
+
+variable "externalservices_prometheus_host" {
+  type    = string
+  default     = null
+}
+
+variable "externalservices_prometheus_basicauth_username" {
+  default     = null
+  type    = string
+  sensitive   = true
+}
+
+variable "externalservices_prometheus_basicauth_password" {
+  default     = null
+  type    = string
+  sensitive   = true
+}
+
+variable "externalservices_loki_host" {
+  default     = null
+  type    = string
+  sensitive   = true
+}
+
+variable "externalservices_loki_basicauth_username" {
+  default     = null
+  type    = string
+  sensitive   = true
+}
+
+variable "externalservices_loki_basicauth_password" {
+  default     = null
+  type    = string
+  sensitive   = true
 }