feat(docker): add full grafana, prometheus, tempo setup for local dev

This is a very useful stack for local development as well, by allowing
to easily access a local grafana instance and look at the accessed
endpoints, query usage and durations etc.
As a nice side effect this also makes sure we have an easy way to
actually test any changes to the opentelemetry integration in an actual
environment instead of just listening to a raw socket.

conf/config.defaults

@@ -179,4 +179,4 @@ expiry_time_statistics = 300
 expiry_time_rss = 300
 
 [tracing]
-otlp_endpoint = http://localhost:4318
+otlp_endpoint = http://localhost:4318/v1/traces

conf/config.dev

@@ -73,3 +73,6 @@ pkgnames-repo = pkgnames.git
 
 [aurblup]
 db-path = YOUR_AUR_ROOT/aurblup/
+
+[tracing]
+otlp_endpoint = http://tempo:4318/v1/traces

docker-compose.yml

@@ -293,6 +293,56 @@ services:
       - ./test:/aurweb/test
       - ./templates:/aurweb/templates
 
+  grafana:
+    # TODO: check if we need init: true
+    image: grafana/grafana:11.1.3
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_LOG_LEVEL=warn
+    # check if depends ar ecorrect, does stopping or restarting a child exit grafana?
+    depends_on:
+      prometheus:
+        condition: service_healthy
+      tempo:
+        condition: service_healthy
+    ports:
+      - "127.0.0.1:3000:3000"
+    volumes:
+      - ./docker/config/grafana/datasources:/etc/grafana/provisioning/datasources
+
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --web.enable-remote-write-receiver
+      - --web.listen-address=prometheus:9090
+    healthcheck:
+      # TODO: check if there is a status route
+      test: "sh /docker/health/prometheus.sh"
+      interval: 3s
+    ports:
+      - "127.0.0.1:9090:9090"
+    volumes:
+      - ./docker/config/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./docker/health/prometheus.sh:/docker/health/prometheus.sh
+
+  tempo:
+    image: grafana/tempo:2.5.0
+    command:
+      - -config.file=/etc/tempo/config.yml
+    healthcheck:
+      # TODO: check if there is a status route
+      test: "sh /docker/health/tempo.sh"
+      interval: 3s
+    ports:
+      - "127.0.0.1:3200:3200"
+      - "127.0.0.1:4318:4318"
+    volumes:
+      - ./docker/config/tempo.yml:/etc/tempo/config.yml
+      - ./docker/health/tempo.sh:/docker/health/tempo.sh
+
 volumes:
   mariadb_test_run: {}
   mariadb_run: {} # Share /var/run/mysqld/mysqld.sock

docker/config/grafana/datasources/datasource.yml

@@ -0,0 +1,42 @@
+---
+apiVersion: 1
+
+deleteDatasources:
+  - name: Prometheus
+  - name: Tempo
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    orgId: 1
+    editable: false
+    jsonData:
+      timeInterval: 1m
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    orgId: 1
+    editable: false
+    jsonData:
+      tracesToMetrics:
+        datasourceUid: 'prometheus'
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+      serviceMap:
+        datasourceUid: 'prometheus'
+      nodeGraph:
+        enabled: true
+      search:
+        hide: false
+      traceQuery:
+        timeShiftEnabled: true
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+      spanBar:
+        type: 'Tag'
+        tag: 'http.path'

docker/config/prometheus.yml

@@ -0,0 +1,15 @@
+---
+global:
+  scrape_interval: 60s
+
+scrape_configs:
+  - job_name: tempo
+    static_configs:
+      - targets: ['tempo:3200']
+        labels:
+          instance: tempo
+  - job_name: aurweb
+    static_configs:
+      - targets: ['fastapi:8000']
+        labels:
+          instance: aurweb

docker/config/tempo.yml

@@ -0,0 +1,54 @@
+---
+stream_over_http_enabled: true
+server:
+  http_listen_address: tempo
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: tempo:4318
+  log_received_spans:
+    enabled: false
+  metric_received_spans:
+    enabled: false
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 1h
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /tmp/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /tmp/tempo/wal
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics, local-blocks]

docker/health/prometheus.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec wget -q http://prometheus:9090/status -O /dev/null

docker/health/tempo.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec wget -q http://tempo:3200/status -O /dev/null