diff --git a/README.md b/README.md index b1211a0f..903535f1 100644 --- a/README.md +++ b/README.md @@ -173,9 +173,19 @@ Now, you can log in to [Grafana Cloud](https://grafana.com/products/cloud/) and To find the labels applied to the telemetry data, refer to [cloud.alloy](./alloy/cloud.alloy) and [docker-compose-cloud.yaml](./docker-compose-cloud.yaml) . -### Monitor frontend with Grafana Cloud Frontend Observability +### Monitor QuickPizza with Grafana Cloud Application and Frontend Observability -To enable [Grafana Cloud Frontend Observability](https://grafana.com/docs/grafana-cloud/monitor-applications/frontend-observability/) for QuickPizza: +The Docker Compose setup is fully instrumented out of the box, so you can jump right into Grafana Cloud Observability apps and start observing the inner workings of the QuickPizza service components. + +To enable [Grafana Cloud Application Observability](https://grafana.com/docs/grafana-cloud/monitor-applications/application-observability/) for QuickPizza: + +1. In your Grafana Cloud instance, navigate to **Observability > Application**. +2. Click on **Enable metrics generation** to enable the usage of Application Observability. +3. Interact with the QuickPizza app to generate traffic. After a few minutes, the QuickPizza components will be automatically discovered and displayed in the UI. + +![Application Observability](./docs/images/grafana-cloud-application-application-observability-quickpizza.png) + +To enable [Grafana Cloud Frontend Observability](https://grafana.com/docs/grafana-cloud/monitor-applications/frontend-observability/): 1. In Grafana Cloud, create a new Frontend Observability application and set the domain to `http://localhost:3333`. 2. Copy the application's Faro web URL. diff --git a/alloy/cloud.alloy b/alloy/cloud.alloy index 507803d5..a9c248de 100644 --- a/alloy/cloud.alloy +++ b/alloy/cloud.alloy @@ -9,17 +9,30 @@ otelcol.receiver.otlp "default" { } output { - traces = [otelcol.processor.batch.default.input] + traces = [ + otelcol.processor.batch.default.input, + otelcol.connector.host_info.default.input, + ] + } +} + +otelcol.connector.host_info "default" { + host_identifiers = ["container.name", "container.id", "service.name"] + metrics_flush_interval = "10s" + + output { + metrics = [otelcol.exporter.prometheus.otlp_metrics.input] } } +otelcol.exporter.prometheus "otlp_metrics" { + forward_to = [ + grafana_cloud.stack.receivers.metrics, + ] +} otelcol.processor.batch "default" { output { - metrics = [] - logs = [] - traces = [ - grafana_cloud.stack.receivers.traces, - ] + traces = [grafana_cloud.stack.receivers.traces] } } @@ -33,6 +46,8 @@ prometheus.scrape "default" { "__address__" = env("QUICKPIZZA_HOST"), "job" = "quickpizza", "instance" = "local", + "service_name" = "quickpizza", + "service_namespace" = "quickpizza", }, ] } @@ -65,6 +80,10 @@ loki.source.docker "default" { host = "unix:///var/run/docker.sock" targets = discovery.relabel.quickpizza.output forward_to = [grafana_cloud.stack.receivers.logs] + labels = { + service_name = "quickpizza", + service_namespace = "quickpizza", + } } import.git "grafana_cloud" { diff --git a/alloy/local-tempo.yaml b/alloy/local-tempo.yaml new file mode 100644 index 00000000..bb07d95f --- /dev/null +++ b/alloy/local-tempo.yaml @@ -0,0 +1,57 @@ +stream_over_http_enabled: true +server: + http_listen_port: 3200 + log_level: info + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + metadata_slo: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: "tempo:4317" + http: + endpoint: "tempo:4318" + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + traces_storage: + path: /var/tempo/generator/traces + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /var/tempo/wal # where to store the wal locally + local: + path: /var/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator + generate_native_histograms: both \ No newline at end of file diff --git a/alloy/local.alloy b/alloy/local.alloy index 50fed8dd..b84749ff 100644 --- a/alloy/local.alloy +++ b/alloy/local.alloy @@ -25,6 +25,7 @@ otelcol.processor.batch "default" { // Scrape Metrics prometheus.scrape "default" { + scrape_interval = "60s" forward_to = [ prometheus.remote_write.default.receiver, ] @@ -33,6 +34,8 @@ prometheus.scrape "default" { "__address__" = env("QUICKPIZZA_HOST"), "job" = "quickpizza", "instance" = "local", + "service_name" = "quickpizza", + "service_namespace" = "quickpizza", }, ] } @@ -86,6 +89,10 @@ loki.source.docker "default" { host = "unix:///var/run/docker.sock" targets = discovery.relabel.quickpizza.output forward_to = [loki.write.local.receiver] + labels = { + service_name = "quickpizza", + service_namespace = "quickpizza", + } } loki.write "local" { diff --git a/docker-compose-cloud.yaml b/docker-compose-cloud.yaml index 713b3e3f..d200d563 100644 --- a/docker-compose-cloud.yaml +++ b/docker-compose-cloud.yaml @@ -9,19 +9,18 @@ services: QUICKPIZZA_OTLP_ENDPOINT: http://alloy:4318 QUICKPIZZA_TRUST_CLIENT_TRACEID: 1 QUICKPIZZA_PYROSCOPE_ENDPOINT: http://alloy:9999 + # Set to 1 to use component name as OpenTelemetry service name + # QUICKPIZZA_OTEL_SERVICE_NAME_LEGACY: 1 # must be set with an .env file QUICKPIZZA_CONF_FARO_URL: "${QUICKPIZZA_CONF_FARO_URL}" # must be set with an .env file QUICKPIZZA_CONF_FARO_APP_NAME: "${QUICKPIZZA_CONF_FARO_APP_NAME}" - + # OTEL service.instance.id label. Default: local + # QUICKPIZZA_OTEL_SERVICE_INSTANCE_ID: local # Namespace label in Faro. Default: quickpizza # QUICKPIZZA_CONF_FARO_APP_NAMESPACE: quickpizza # Enable logging. Possible values: error, warn, debug. Default: info # QUICKPIZZA_LOG_LEVEL: debug - # Service name label in Pyroscope. Default: quickpizza - # QUICKPIZZA_PYROSCOPE_NAME: quickpizza - # Namespace label in Pyroscope. Default: quickpizza - # QUICKPIZZA_PYROSCOPE_NAMESPACE: quickpizza alloy: image: grafana/alloy:v1.9.1 diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml index 3857e2ab..24565a91 100644 --- a/docker-compose-local.yaml +++ b/docker-compose-local.yaml @@ -11,12 +11,6 @@ services: QUICKPIZZA_PYROSCOPE_ENDPOINT: http://alloy:9999 # Enable logging. Possible values: error, warn, debug. Default: info # QUICKPIZZA_LOG_LEVEL: debug - # Service name label in Pyroscope. Default: quickpizza - # QUICKPIZZA_PYROSCOPE_NAME: quickpizza - # Namespace label in Pyroscope. Default: quickpizza - # QUICKPIZZA_PYROSCOPE_NAMESPACE: quickpizza - # Database configuration (default: in-memory SQLite) - # QUICKPIZZA_DB: postgres://user:password@postgres:5432/quickpizza?sslmode=disable loki: image: grafana/loki:3.4.3 @@ -27,6 +21,7 @@ services: image: prom/prometheus:v3.2.1 command: - --web.enable-remote-write-receiver + - --enable-feature=exemplar-storage - --enable-feature=native-histograms - --config.file=/etc/prometheus/prometheus.yml ports: @@ -36,22 +31,21 @@ services: image: grafana/pyroscope:1.14.0 ports: - "4040:4040" - + tempo: image: grafana/tempo:2.8.1 - command: - - "-storage.trace.backend=local" # tell tempo where to permanently put traces - - "-storage.trace.local.path=/tmp/tempo/traces" - - "-storage.trace.wal.path=/tmp/tempo/wal" # tell tempo where to store the wal - - "-auth.enabled=false" # disables the requirement for the X-Scope-OrgID header - - "-server.http-listen-port=3200" + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - ./alloy/local-tempo.yaml:/etc/tempo.yaml ports: - "3200:3200" - "4317:4317" - "4318:4318" + depends_on: + - prometheus grafana: - image: grafana/grafana:${GRAFANA_VERSION:-12.0.2} + image: grafana/grafana:${GRAFANA_VERSION:-12.1.0} ports: - "3000:3000" environment: diff --git a/docs/images/grafana-cloud-application-application-observability-quickpizza.png b/docs/images/grafana-cloud-application-application-observability-quickpizza.png new file mode 100644 index 00000000..2e54c4b3 Binary files /dev/null and b/docs/images/grafana-cloud-application-application-observability-quickpizza.png differ diff --git a/grafana/datasources/datasource.yaml b/grafana/datasources/datasource.yaml index 67b63c61..63aefa81 100644 --- a/grafana/datasources/datasource.yaml +++ b/grafana/datasources/datasource.yaml @@ -14,6 +14,7 @@ datasources: jsonData: tlsAuth: false tlsAuthWithCACert: false + timeInterval: 60s # Scrape interval (should match Prometheus configuration, defined in local.alloy) editable: false - name: Tempo type: tempo diff --git a/pkg/http/tracing.go b/pkg/http/tracing.go index e4e4004e..6d2ae753 100644 --- a/pkg/http/tracing.go +++ b/pkg/http/tracing.go @@ -95,6 +95,15 @@ func (t *TraceInstaller) Install(r chi.Router, serviceName string, extraOpts ... } } + serviceNamespace, ok := os.LookupEnv("QUICKPIZZA_OTEL_SERVICE_NAMESPACE") + if !ok { + serviceNamespace = "quickpizza" + } + serviceInstanceID, ok := os.LookupEnv("QUICKPIZZA_OTEL_SERVICE_INSTANCE_ID") + if !ok { + serviceInstanceID = "local" + } + // We discard the error here as it cannot possibly take place with the parameters we use. res, _ := resource.Merge( resource.Default(), @@ -102,6 +111,8 @@ func (t *TraceInstaller) Install(r chi.Router, serviceName string, extraOpts ... semconv.SchemaURL, semconv.ServiceName(serviceNameAttrValue), attribute.KeyValue{Key: "service.component", Value: attribute.StringValue(serviceName)}, + attribute.KeyValue{Key: "service.namespace", Value: attribute.StringValue(serviceNamespace)}, + attribute.KeyValue{Key: "service.instance.id", Value: attribute.StringValue(serviceInstanceID)}, ), )