diff --git a/backend/apps/ifc_validation/tasks/check_programs.py b/backend/apps/ifc_validation/tasks/check_programs.py index af50dd1b..4ffc9048 100644 --- a/backend/apps/ifc_validation/tasks/check_programs.py +++ b/backend/apps/ifc_validation/tasks/check_programs.py @@ -3,7 +3,8 @@ import json import shutil import subprocess -from typing import List +import psutil +from typing import List, Optional from dataclasses import dataclass # pip install filetype @@ -23,11 +24,43 @@ class proc_output: stdout : str stderr : str args: List[str] + peak_rss_kb : Optional[int] = None + min_mem_available_kb : Optional[int] = None + + +def _read_proc_kb(path, field): + # read a ": kB" line from a /proc file (Linux only) + try: + with open(path) as f: + for line in f: + if line.startswith(field + ":"): + return int(line.split()[1]) + except (OSError, ValueError, IndexError): + pass + return None + + +def _read_peak_rss_kb(pid): + # summed over the process tree: gherkin spawns behave as a nested child, so the + # direct child is only a thin orchestrator while behave holds the parsed model + total = _read_proc_kb(f"/proc/{pid}/status", "VmHWM") + if total is None: + return None + try: + for child in psutil.Process(pid).children(recursive=True): + hwm = _read_proc_kb(f"/proc/{child.pid}/status", "VmHWM") + if hwm: + total += hwm + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + return total def run_subprocess_wait(*popen_args, check=False, **popen_kwargs): process = subprocess.Popen(*popen_args, **popen_kwargs) out_chunks, err_chunks = [], [] + peak_rss_kb = None + min_mem_available_kb = None try: while True: try: @@ -37,6 +70,14 @@ def run_subprocess_wait(*popen_args, check=False, **popen_kwargs): break except subprocess.TimeoutExpired: # keep looping; you can also check your own stop conditions here + # tree sum is not monotonic (children exit), so keep the max + sample = _read_peak_rss_kb(process.pid) + if sample is not None: + peak_rss_kb = sample if peak_rss_kb is None else max(peak_rss_kb, sample) + # lowest MemAvailable seen = tightest moment during this run + mem_available = _read_proc_kb("/proc/meminfo", "MemAvailable") + if mem_available is not None: + min_mem_available_kb = mem_available if min_mem_available_kb is None else min(min_mem_available_kb, mem_available) continue except BaseException as e: process.terminate() @@ -50,7 +91,7 @@ def run_subprocess_wait(*popen_args, check=False, **popen_kwargs): stdout, stderr = "".join(out_chunks), "".join(err_chunks) if check and retcode != 0: raise subprocess.CalledProcessError(retcode, popen_args[0], output=stdout, stderr=stderr) - return proc_output(retcode, stdout, stderr, popen_args[0] if popen_args else []) + return proc_output(retcode, stdout, stderr, popen_args[0] if popen_args else [], peak_rss_kb, min_mem_available_kb) checks_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "checks")) @@ -145,10 +186,17 @@ def check_magic_and_clamav(context:TaskContext): '' if (scanner == clamdscan) else f'--max-filesize={MAX_FILE_SIZE_IN_MB}M', context.file_path] ) - if proc.returncode != 0: + if proc.returncode == 1: + # rc 1 = virus found result = { 'invalid': f'suspicious file\n\n{proc.stdout}\n{proc.stderr}' } + elif proc.returncode != 0: + # rc >= 2 = scanner failure (e.g. clamd down): fail the task and leave + # the file alone; a broken scanner is not an infected file + result = { + 'error': f'scanner error (rc={proc.returncode})\n\n{proc.stdout}\n{proc.stderr}' + } else: result = { 'valid': 'unknown type' if ty is None else ty.mime @@ -310,6 +358,12 @@ def run_subprocess( env= os.environ.copy() ) logger.info(f'test run task task name {task.type}, task value : {task}') + if proc.peak_rss_kb is not None: + logger.info( + f'Peak RSS for {task.type} subprocess (task #{task.id}): {proc.peak_rss_kb} kB ' + f'(min MemAvailable during run: {proc.min_mem_available_kb} kB, ' + f'worker RSS: {_read_proc_kb("/proc/self/status", "VmRSS")} kB)' + ) return proc except Exception as err: diff --git a/backend/apps/ifc_validation/tests/tests_subprocess_peak_rss.py b/backend/apps/ifc_validation/tests/tests_subprocess_peak_rss.py new file mode 100644 index 00000000..e155aadb --- /dev/null +++ b/backend/apps/ifc_validation/tests/tests_subprocess_peak_rss.py @@ -0,0 +1,81 @@ +import sys +import subprocess + +from django.test import SimpleTestCase, TransactionTestCase +from django.contrib.auth.models import User + +from apps.ifc_validation_models.models import ValidationRequest, ValidationTask, set_user_context + +from ..tasks.check_programs import run_subprocess_wait, run_subprocess + + +class SubprocessPeakRssTestCase(SimpleTestCase): + + def test_peak_rss_captured_for_memory_hungry_subprocess(self): + # allocate ~100 MB and stay alive long enough for the 0.2s poll to sample it + child = "data = bytearray(100 * 1024 * 1024)\nimport time\ntime.sleep(0.6)" + proc = run_subprocess_wait( + [sys.executable, "-c", child], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + self.assertEqual(proc.returncode, 0) + self.assertIsNotNone(proc.peak_rss_kb) + self.assertGreater(proc.peak_rss_kb, 100 * 1024) + self.assertIsNotNone(proc.min_mem_available_kb) + self.assertGreater(proc.min_mem_available_kb, 0) + + def test_fast_subprocess_still_succeeds_without_peak_sample(self): + # a subprocess that exits before the first 0.2s poll simply has no sample + proc = run_subprocess_wait( + [sys.executable, "-c", "print('hi')"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stdout.strip(), "hi") + + def test_peak_rss_includes_nested_child_processes(self): + # gherkin spawns behave as a nested child, so grandchildren must be counted: + # thin direct child, nested child allocates ~150 MB + child = ( + "import subprocess, sys\n" + "subprocess.run([sys.executable, '-c', " + "'data = bytearray(150 * 1024 * 1024)\\nimport time\\ntime.sleep(0.8)'])\n" + ) + proc = run_subprocess_wait( + [sys.executable, "-c", child], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + self.assertEqual(proc.returncode, 0) + self.assertIsNotNone(proc.peak_rss_kb) + self.assertGreater(proc.peak_rss_kb, 150 * 1024) + + def test_failing_subprocess_still_reports_returncode(self): + proc = run_subprocess_wait( + [sys.executable, "-c", "import sys; sys.exit(3)"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + ) + self.assertEqual(proc.returncode, 3) + + +class RunSubprocessTaskLoggingTestCase(TransactionTestCase): + + def test_peak_rss_is_logged_for_a_real_validation_task(self): + user, _ = User.objects.get_or_create(id=1, defaults={'username': 'SYSTEM', 'is_active': True}) + set_user_context(user) + request = ValidationRequest.objects.create( + file_name='wall-with-opening-and-window.ifc', + file='wall-with-opening-and-window.ifc', + size=12789 + ) + task = ValidationTask.objects.create(request=request, type=ValidationTask.Type.SYNTAX) + + child = "data = bytearray(50 * 1024 * 1024)\nimport time\ntime.sleep(0.5)" + with self.assertLogs('ifc_validation', level='INFO') as captured: + proc = run_subprocess(task, [sys.executable, "-c", child]) + + self.assertEqual(proc.returncode, 0) + peak_lines = [line for line in captured.output if 'Peak RSS for' in line] + self.assertEqual(len(peak_lines), 1) + self.assertIn(f'task #{task.id}', peak_lines[0]) + self.assertIn('min MemAvailable during run', peak_lines[0]) + self.assertIn('worker RSS', peak_lines[0]) diff --git a/docker-compose.swarm.nodb.yml b/docker-compose.swarm.nodb.yml index ffc47143..e6403ecc 100644 --- a/docker-compose.swarm.nodb.yml +++ b/docker-compose.swarm.nodb.yml @@ -1,158 +1,276 @@ -# Docker Swarm deployment configuration — external database (no containerized PostgreSQL) -# -# Usage: -# make start-swarm-nodb ENV_FILE=.env.DEV_SWARM -# -# Same as docker-compose.swarm.yml but without the db service. -# Set POSTGRES_HOST, POSTGRES_PORT, etc. in your env file to point to the external DB. - -services: - - frontend: - image: ${REGISTRY}/validationsvc-frontend - ports: - - 80:80 - - 443:443 - environment: - CERTBOT_DOMAIN: ${CERTBOT_DOMAIN} - CERTBOT_EMAIL: ${CERTBOT_EMAIL} - volumes: - - letsencrypt_data:/etc/letsencrypt - - static_data:/app/backend/django_static - networks: - - validate - deploy: - replicas: 1 - placement: - constraints: [node.role == manager] - restart_policy: - condition: on-failure - delay: 5s - - backend: - image: ${REGISTRY}/validationsvc-backend - entrypoint: /app/backend/server-entrypoint.sh - env_file: ${ENV_FILE} - volumes: - - static_data:/app/backend/django_static - - files_data:/files_storage - - gherkin_rules_log_data:/gherkin_logs - expose: - - 8000 - networks: - - validate - deploy: - replicas: 1 - placement: - constraints: [node.role == manager] - restart_policy: - condition: on-failure - delay: 5s - update_config: - parallelism: 1 - delay: 30s - failure_action: rollback - healthcheck: - test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/api/')\""] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - worker: - image: ${REGISTRY}/validationsvc-backend - entrypoint: /app/backend/worker-entrypoint.sh - env_file: ${ENV_FILE} - volumes: - - files_data:/files_storage - - gherkin_rules_log_data:/gherkin_logs - networks: - - validate - deploy: - replicas: 2 - # No placement constraint - workers run on any node - restart_policy: - condition: on-failure - delay: 5s - update_config: - parallelism: 1 - delay: 30s - failure_action: rollback - - av_worker: - image: ${REGISTRY}/validationsvc-backend - entrypoint: /app/backend/worker-clamav-entrypoint.sh - env_file: ${ENV_FILE} - volumes: - - files_data:/files_storage - - clamav_data:/var/lib/clamav # persist databa - networks: - - validate - deploy: - replicas: 1 - # No placement constraint - workers run on any node - restart_policy: - condition: on-failure - delay: 5s - update_config: - parallelism: 1 - delay: 30s - failure_action: rollback - - scheduler: - image: ${REGISTRY}/validationsvc-backend - entrypoint: /app/backend/worker-beat-entrypoint.sh - env_file: ${ENV_FILE} - volumes: - - files_data:/files_storage - - gherkin_rules_log_data:/gherkin_logs - networks: - - validate - deploy: - replicas: 1 - placement: - constraints: [node.role == manager] - restart_policy: - condition: on-failure - delay: 5s - - redis: - image: redis:8.4-alpine - command: redis-server --protected-mode no --bind 0.0.0.0 - expose: - - 6379 - volumes: - - redis_data:/data - networks: - - validate - deploy: - replicas: 1 - placement: - constraints: [node.role == manager] - restart_policy: - condition: on-failure - delay: 5s - -networks: - validate: - driver: overlay - driver_opts: - com.docker.network.driver.mtu: "1400" - -volumes: - static_data: - letsencrypt_data: - redis_data: - files_data: - driver: local - driver_opts: - type: nfs - o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2" - device: ":/srv/nfs/files_data" - gherkin_rules_log_data: - driver: local - driver_opts: - type: nfs - o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2" - device: ":/srv/nfs/gherkin_logs" - clamav_data: \ No newline at end of file +# Docker Swarm deployment configuration — external database (no containerized PostgreSQL) +# +# Usage: +# make start-swarm-nodb ENV_FILE=.env.DEV_SWARM +# +# Same as docker-compose.swarm.yml but without the db service. +# Set POSTGRES_HOST, POSTGRES_PORT, etc. in your env file to point to the external DB. + +services: + + frontend: + image: ${REGISTRY}/validationsvc-frontend + ports: + - 80:80 + - 443:443 + environment: + CERTBOT_DOMAIN: ${CERTBOT_DOMAIN} + CERTBOT_EMAIL: ${CERTBOT_EMAIL} + volumes: + - letsencrypt_data:/etc/letsencrypt + - static_data:/app/backend/django_static + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + + backend: + image: ${REGISTRY}/validationsvc-backend + entrypoint: /app/backend/server-entrypoint.sh + env_file: ${ENV_FILE} + volumes: + - static_data:/app/backend/django_static + - files_data:/files_storage + - gherkin_rules_log_data:/gherkin_logs + expose: + - 8000 + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + update_config: + parallelism: 1 + delay: 30s + failure_action: rollback + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8000/api/')\""] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + worker: + image: ${REGISTRY}/validationsvc-backend + entrypoint: /app/backend/worker-entrypoint.sh + env_file: ${ENV_FILE} + volumes: + - files_data:/files_storage + - gherkin_rules_log_data:/gherkin_logs + networks: + - validate + deploy: + replicas: 2 + # No placement constraint - workers run on any node + resources: + limits: + memory: ${WORKER_MEMORY_LIMIT} + reservations: + memory: ${WORKER_MEMORY_RESERVATION} + restart_policy: + condition: on-failure + delay: 5s + update_config: + parallelism: 1 + delay: 30s + failure_action: rollback + + av_worker: + image: ${REGISTRY}/validationsvc-backend + entrypoint: /app/backend/worker-clamav-entrypoint.sh + env_file: ${ENV_FILE} + volumes: + - files_data:/files_storage + - clamav_data:/var/lib/clamav # persist databa + networks: + - validate + deploy: + replicas: 1 + resources: + limits: + memory: 4G + # No placement constraint - workers run on any node + restart_policy: + condition: on-failure + delay: 5s + update_config: + parallelism: 1 + delay: 30s + failure_action: rollback + + scheduler: + image: ${REGISTRY}/validationsvc-backend + entrypoint: /app/backend/worker-beat-entrypoint.sh + env_file: ${ENV_FILE} + volumes: + - files_data:/files_storage + - gherkin_rules_log_data:/gherkin_logs + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + + redis: + image: redis:8.4-alpine + command: redis-server --protected-mode no --bind 0.0.0.0 + expose: + - 6379 + volumes: + - redis_data:/data + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + + # --------------------------------------------------------------------------- + # Observability (additive services; removing them restores the previous stack). + # Config files are bind-mounted from the checkout via ${PWD}, which envsubst + # fills in at deploy time. That is why these services are pinned to the manager + # node: that is where the checkout lives. + # --------------------------------------------------------------------------- + + otel_col: + image: otel/opentelemetry-collector-contrib:0.156.0 + command: ["--config=/etc/otel-collector-config.yaml"] + expose: + - 4317 # OTLP over gRPC receiver + - 4318 # OTLP over HTTP receiver + - 8888 # collector self-metrics (Prometheus scrape) + - 8889 # prometheus exporter: OTLP-received app metrics + - 13133 # health_check extension + volumes: + - ${PWD}/docker/otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + + prometheus: + image: prom/prometheus:v3.13.1 + command: + - '--config.file=/etc/prometheus/prometheus.yml' + # Keep one year of history instead of the 15-day default. Quarterly trends + # (file sizes, memory usage) are exactly what we need and 15 days erases + # them. The time-series database was only 162 MB at 15 days, so a year is + # cheap in storage terms. + - '--storage.tsdb.retention.time=1y' + expose: + - 9090 + volumes: + - ${PWD}/docker/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + + grafana: + image: grafana/grafana:12.3.9 + # env_file gives Grafana POSTGRES_HOST/PORT/USER/NAME/PASSWORD for the + # provisioned datasource (provisioning/datasources/postgres.yaml). + # Same pattern as backend and worker. It also avoids envsubst problems: + # variables not listed in the Makefile's SWARM_VARS would be substituted + # with an empty string. + # NOTE: set GF_SECURITY_ADMIN_PASSWORD in the environment file, otherwise + # Grafana starts with the default admin/admin credentials. + env_file: ${ENV_FILE} + ports: + - 3000:3000 + volumes: + - ${PWD}/docker/grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ${PWD}/docker/grafana/provisioning:/etc/grafana/provisioning:ro + # dashboards as code, loaded via provisioning/dashboards/vs-dashboards.yaml + - ${PWD}/docker/grafana/dashboards:/etc/grafana/dashboards-vs:ro + - grafana_data:/var/lib/grafana + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + + node_exporter: + image: prom/node-exporter:v1.12.1 + command: + - '--path.rootfs=/host' + volumes: + - /:/host:ro,rslave + networks: + - validate + deploy: + mode: global # one per node; Prometheus discovers via tasks.node_exporter + restart_policy: + condition: on-failure + delay: 5s + + celery_exporter: + image: danihodovic/celery-exporter:0.12.2 + environment: + CE_BROKER_URL: redis://redis:6379/0 + expose: + - 9808 + networks: + - validate + deploy: + replicas: 1 + restart_policy: + condition: on-failure + delay: 5s + +networks: + validate: + driver: overlay + driver_opts: + com.docker.network.driver.mtu: "1400" + +volumes: + static_data: + letsencrypt_data: + redis_data: + files_data: + driver: local + driver_opts: + type: nfs + o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2" + device: ":/srv/nfs/files_data" + gherkin_rules_log_data: + driver: local + driver_opts: + type: nfs + o: "addr=${NFS_SERVER_IP},nfsvers=4.1,rw,hard,timeo=600,retrans=2" + device: ":/srv/nfs/gherkin_logs" + clamav_data: + prometheus_data: + grafana_data: \ No newline at end of file diff --git a/docker/frontend/nginx/default.conf.template b/docker/frontend/nginx/default.conf.template index 7c3bbf25..f7b9af3d 100644 --- a/docker/frontend/nginx/default.conf.template +++ b/docker/frontend/nginx/default.conf.template @@ -1,113 +1,141 @@ -server { - listen 80; - server_name ${CERTBOT_DOMAIN}; - server_tokens off; - - # turn off buffering, enable streaming to backend - proxy_request_buffering off; - proxy_http_version 1.1; - client_max_body_size 0; # turn off buffer/max size - - # turn on compression - gzip on; - gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json; - gzip_proxied no-cache no-store private expired auth; - gzip_min_length 1024; - - # security headers - add_header X-Content-Type-Options nosniff; - add_header X-Frame-Options DENY always; - # NB: 'same-origin' instead of 'no-referrer' — the backend relies on the Referer header - # to distinguish WEBUI vs API uploads (see views_legacy.py). 'no-referrer' strips it - # entirely, causing a 500 on file upload. 'same-origin' still prevents leaking the - # referer to third parties (satisfying pentest requirements). - add_header Referrer-Policy 'same-origin'; - add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://code.jquery.com; frame-ancestors 'none'"; - - # React UI - location / { - root /app/frontend; - index index.html index.htm; - try_files $uri $uri/ /index.html; - } - - # BFF (React UI) - location /bff { - try_files $uri @proxy_api; - } - - # Swagger/ReDoc — relaxed CSP for inline scripts, proxied directly - location ~ ^/api/v1/(swagger-ui|redoc) { - resolver 127.0.0.11 valid=30s; - set $upstream http://backend:8000; - add_header X-Content-Type-Options nosniff always; - add_header X-Frame-Options DENY always; - add_header Referrer-Policy 'same-origin' always; - add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; frame-ancestors 'none'" always; - proxy_read_timeout 500s; - proxy_connect_timeout 75s; - proxy_pass $upstream; - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # API (Django APP) - location /api { - try_files $uri @proxy_api; - } - - # Django admin - location /admin { - try_files $uri @proxy_api; - } - - # Django admin files - location /files { - try_files $uri @proxy_api; - } - - # Django SQL Explorer - location /sqlexplorer { - try_files $uri @proxy_api; - } - - # AD B2C - location /login { - try_files $uri @proxy_api; - } - location /whoami { - try_files $uri @proxy_api; - } - location /logout { - try_files $uri @proxy_api; - } - location /callback { - try_files $uri @proxy_api; - } - - # Django backend (API + BFF + Admin) - location @proxy_api { - resolver 127.0.0.11 valid=30s; - set $upstream http://backend:8000; - - proxy_read_timeout 500s; - proxy_connect_timeout 75s; - - proxy_pass $upstream; - proxy_redirect off; - - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # static files - location /django_static/ { - autoindex on; - alias /app/backend/django_static/; - } -} +server { + listen 80; + server_name ${CERTBOT_DOMAIN}; + server_tokens off; + + # turn off buffering, enable streaming to backend + proxy_request_buffering off; + proxy_http_version 1.1; + client_max_body_size 0; # turn off buffer/max size + + # turn on compression + gzip on; + gzip_types text/plain application/javascript application/x-javascript text/javascript text/xml text/css application/json; + gzip_proxied no-cache no-store private expired auth; + gzip_min_length 1024; + + # security headers + add_header X-Content-Type-Options nosniff; + add_header X-Frame-Options DENY always; + # NB: 'same-origin' instead of 'no-referrer' — the backend relies on the Referer header + # to distinguish WEBUI vs API uploads (see views_legacy.py). 'no-referrer' strips it + # entirely, causing a 500 on file upload. 'same-origin' still prevents leaking the + # referer to third parties (satisfying pentest requirements). + add_header Referrer-Policy 'same-origin'; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://code.jquery.com; frame-ancestors 'none'"; + + # React UI + location / { + root /app/frontend; + index index.html index.htm; + try_files $uri $uri/ /index.html; + } + + # Grafana dashboards, served behind the same nginx as the Django admin. + # Grafana serves itself from /grafana/ because of root_url and + # serve_from_sub_path in docker/grafana/grafana.ini. It has its own login + # (GF_SECURITY_ADMIN_PASSWORD). + location /grafana/ { + # Important: resolve the upstream through a VARIABLE plus a resolver, not as a + # hardcoded host name. Without this, nginx resolves the name at startup and + # refuses to start the ENTIRE site whenever Grafana happens to be down. + # With a variable, only /grafana/ fails. 127.0.0.11 is Docker's internal DNS. + # Grafana's table panels use new Function() to render cells. The global CSP + # (around line 25) blocks that with "call to Function() blocked by CSP". + # This add_header replaces the inherited CSP for this location only. + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval'; frame-ancestors 'none'" always; + resolver 127.0.0.11 valid=30s ipv6=off; + set $grafana_upstream http://grafana:3000; + # No path rewriting: Grafana serves itself from /grafana/ + # (serve_from_sub_path = true in grafana.ini). + proxy_pass $grafana_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # live tail / websockets in Grafana + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # BFF (React UI) + location /bff { + try_files $uri @proxy_api; + } + + # Swagger/ReDoc — relaxed CSP for inline scripts, proxied directly + location ~ ^/api/v1/(swagger-ui|redoc) { + resolver 127.0.0.11 valid=30s; + set $upstream http://backend:8000; + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy 'same-origin' always; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; frame-ancestors 'none'" always; + proxy_read_timeout 500s; + proxy_connect_timeout 75s; + proxy_pass $upstream; + proxy_redirect off; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # API (Django APP) + location /api { + try_files $uri @proxy_api; + } + + # Django admin + location /admin { + try_files $uri @proxy_api; + } + + # Django admin files + location /files { + try_files $uri @proxy_api; + } + + # Django SQL Explorer + location /sqlexplorer { + try_files $uri @proxy_api; + } + + # AD B2C + location /login { + try_files $uri @proxy_api; + } + location /whoami { + try_files $uri @proxy_api; + } + location /logout { + try_files $uri @proxy_api; + } + location /callback { + try_files $uri @proxy_api; + } + + # Django backend (API + BFF + Admin) + location @proxy_api { + resolver 127.0.0.11 valid=30s; + set $upstream http://backend:8000; + + proxy_read_timeout 500s; + proxy_connect_timeout 75s; + + proxy_pass $upstream; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # static files + location /django_static/ { + autoindex on; + alias /app/backend/django_static/; + } +} diff --git a/docker/grafana/dashboards/vs-platform-usage.json b/docker/grafana/dashboards/vs-platform-usage.json new file mode 100644 index 00000000..22567760 --- /dev/null +++ b/docker/grafana/dashboards/vs-platform-usage.json @@ -0,0 +1,285 @@ +{ + "id": null, + "panels": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "format": "time_series", + "rawSql": "SELECT created::date AS time, COUNT(*)::float AS validations FROM ifc_validation_request WHERE created > NOW() - INTERVAL '30 days' GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "title": "Validation requests per day (30d)", + "type": "timeseries", + "description": "Number of files submitted per day (fixed 30-day window, independent of the time range above). Includes requests that were soft-deleted later." + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "targets": [ + { + "format": "table", + "rawSql": "SELECT type, COUNT(*) AS n, ROUND(percentile_cont(0.5) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ended-started))::numeric,1) AS p50_s, ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ended-started))::numeric,1) AS p95_s FROM ifc_validation_task WHERE ended IS NOT NULL AND started IS NOT NULL AND created > NOW() - INTERVAL '90 days' GROUP BY type ORDER BY p95_s DESC", + "refId": "A" + } + ], + "title": "Duration per task type: p50 / p95 (90d, seconds)", + "type": "table", + "description": "Median (p50) and slow-tail (p95) duration in seconds per validation step, over 90 days. From ifc_validation_task.ended - started." + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "targets": [ + { + "format": "time_series", + "rawSql": "SELECT DATE(r.created) AS time, percentile_cont(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ft.fs - r.created)) AS queue_wait_p95_s FROM ifc_validation_request r JOIN LATERAL (SELECT MIN(t.started) AS fs FROM ifc_validation_task t WHERE t.request_id = r.id) ft ON ft.fs IS NOT NULL WHERE r.created > NOW() - INTERVAL '30 days' GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "title": "Queue wait time p95 per day (s, 30d)", + "type": "timeseries", + "description": "Wait time between submission and the start of the first task. NOTE: on DEV, tasks are sometimes re-run manually on old requests, which inflates this to hours. Read it as a trend, not an absolute." + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "id": 4, + "targets": [ + { + "format": "time_series", + "rawSql": "SELECT created::date AS time, ROUND(100.0 * SUM((status='FAILED')::int) / COUNT(*), 1) AS failure_rate FROM ifc_validation_request WHERE created > NOW() - INTERVAL '30 days' GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "title": "Failure rate per day (%, 30d)", + "type": "timeseries", + "description": "Percentage of requests per day that ended in status FAILED." + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 5, + "targets": [ + { + "format": "table", + "rawSql": "SELECT EXTRACT(HOUR FROM created)::int AS hour, COUNT(*) AS requests FROM ifc_validation_request WHERE created > NOW() - INTERVAL '90 days' GROUP BY 1 ORDER BY 1", + "refId": "A" + } + ], + "title": "Activity by hour of day (90d)", + "type": "barchart", + "description": "Which hour of the day the platform is used (UTC), over 90 days." + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 6, + "targets": [ + { + "format": "table", + "rawSql": "SELECT COUNT(*) FROM ifc_validation_request WHERE completed IS NULL AND status NOT IN ('COMPLETED','FAILED') AND created < NOW() - INTERVAL '1 hour'", + "refId": "A" + } + ], + "title": "Stuck requests (>1h, not finished)", + "type": "stat", + "description": "Requests older than one hour that are still not finished (not COMPLETED/FAILED). Should be 0." + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 5, + "w": 18, + "x": 6, + "y": 16 + }, + "id": 7, + "targets": [ + { + "format": "table", + "rawSql": "SELECT file_name, ROUND(size/1024.0/1024.0,1) AS mb, status, EXTRACT(EPOCH FROM completed-created)::int AS duration_s, created FROM ifc_validation_request WHERE created > NOW() - INTERVAL '30 days' ORDER BY size DESC NULLS LAST LIMIT 10", + "refId": "A" + } + ], + "title": "Largest files in the last 30d (top 10)", + "type": "table", + "description": "The ten largest files of the last 30 days. Odd-looking file names are non-Latin names exactly as stored in the database." + }, + { + "id": 8, + "type": "timeseries", + "title": "Uploads per day by channel (API vs WEBUI)", + "description": "NOTE: all rows before 2025-07-24 were written as WEBUI during a migration, so API figures are only reliable from Aug 2025 onwards. Counts uploads only — status polling and other GET traffic exist solely in the nginx logs.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "fieldConfig": { + "defaults": { + "custom": { + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "format": "time_series", + "rawSql": "SELECT created::date AS time, channel, COUNT(*)::float AS uploads FROM ifc_validation_request WHERE created > NOW() - INTERVAL '90 days' AND channel IS NOT NULL GROUP BY 1,2 ORDER BY 1" + } + ] + }, + { + "id": 9, + "type": "table", + "title": "Top API users (90d)", + "description": "Per account: number of uploads, total and average size. Some accounts have no email filled in, hence grouping by username.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "format": "table", + "rawSql": "SELECT u.username, COUNT(*) AS uploads, ROUND(SUM(r.size)/1024.0/1024.0,1) AS total_mb, ROUND(AVG(r.size)/1024.0/1024.0,2) AS avg_mb, MAX(r.created)::date AS last_upload FROM ifc_validation_request r JOIN auth_user u ON u.id = r.created_by_id WHERE r.channel='API' AND r.created > NOW() - INTERVAL '90 days' GROUP BY 1 ORDER BY uploads DESC LIMIT 15" + } + ] + }, + { + "id": 10, + "type": "table", + "title": "Crash causes: why tasks FAIL (not validation errors)", + "description": "These are system failures, not 'the model is invalid'. The first line of status_reason is used as the category. Four recurring types: a duplicate-key race on concurrent requests, NUL bytes that PostgreSQL rejects, and two code bugs (TaskContext missing proc, NoneType has no id).", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 30 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "targets": [ + { + "refId": "A", + "format": "table", + "rawSql": "SELECT CASE WHEN status_reason LIKE '%duplicate key%' THEN 'duplicate key (race on concurrent requests)' WHEN status_reason LIKE '%NUL (0x00)%' THEN 'NUL bytes in text (PostgreSQL rejects)' WHEN status_reason LIKE '%TaskContext%' THEN 'code bug: TaskContext missing proc' WHEN status_reason LIKE '%NoneType%' THEN 'code bug: NoneType has no id' WHEN status_reason IS NULL OR status_reason='' THEN '(no reason recorded)' ELSE split_part(status_reason, E'\\n', 1) END AS cause, COUNT(*) AS count, COUNT(DISTINCT type) AS task_types, MAX(created)::date AS last_seen FROM ifc_validation_task WHERE status='FAILED' GROUP BY 1 ORDER BY aantal DESC LIMIT 15" + } + ] + } + ], + "refresh": "5m", + "tags": [ + "observability", + "ivs-681" + ], + "time": { + "from": "now-30d", + "to": "now" + }, + "title": "Validation Service — Platform Usage (DB)", + "uid": "vs-platform-usage", + "version": 1, + "schemaVersion": 39, + "editable": true, + "timezone": "browser" +} \ No newline at end of file diff --git a/docker/grafana/dashboards/vs-system-health.json b/docker/grafana/dashboards/vs-system-health.json new file mode 100644 index 00000000..0de18c4b --- /dev/null +++ b/docker/grafana/dashboards/vs-system-health.json @@ -0,0 +1,594 @@ +{ + "id": null, + "panels": [ + { + "id": 8, + "type": "stat", + "title": "Alert candidate: antivirus queue", + "description": "E1 threshold: backlog on the antivirus queue > 0. Every validation passes through this queue; the av_worker zombie of 27 July sat dead for 28 hours without anyone noticing.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "expr": "max(celery_queue_length{queue_name=\"antivirus\"})", + "refId": "A", + "legendFormat": "__auto" + } + ] + }, + { + "id": 9, + "type": "stat", + "title": "Alert candidate: memory (min across nodes)", + "description": "E1 threshold: MemAvailable < 2 GB (red; orange below 4 GB). Lowest value across all nodes. Without swap, memory pressure is immediately fatal — that is what made the freeze so abrupt.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 2147483648 + }, + { + "color": "green", + "value": 4294967296 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "expr": "min(node_memory_MemAvailable_bytes)", + "refId": "A", + "legendFormat": "__auto" + } + ] + }, + { + "id": 10, + "type": "stat", + "title": "Alert candidate: disk (max across nodes)", + "description": "E1 threshold: disk > 85% (orange), > 95% red. The fullest node counts. Currently red as expected: dev-vm-worker-1 sits around 96% (IVS-828, backlog item A4).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "expr": "max(100 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} * 100))", + "refId": "A", + "legendFormat": "__auto" + } + ] + }, + { + "id": 11, + "type": "stat", + "title": "Alert candidate: stuck requests (>1h)", + "description": "Requests older than 1 hour that are neither finished nor FAILED — same definition as the panel on Platform Usage, but as a Prometheus metric so it can drive an alert later.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "expr": "max(vs_requests_stuck)", + "refId": "A", + "legendFormat": "__auto" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 1, + "targets": [ + { + "expr": "100 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Disk used % (per node) — the IVS-828 chart", + "type": "timeseries", + "description": "Percentage of disk in use, per Swarm node. Above ~90% things get risky." + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 2, + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Memory available (per node)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 3, + "targets": [ + { + "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "CPU usage % (per node)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 4, + "targets": [ + { + "expr": "celery_active_worker_count", + "legendFormat": "{{queue_name}}", + "refId": "A" + } + ], + "title": "Celery: active workers per queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 5, + "targets": [ + { + "expr": "celery_queue_length", + "legendFormat": "{{queue_name}}", + "refId": "A" + } + ], + "title": "Celery: queue length (backlog)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 6, + "targets": [ + { + "expr": "celery_active_process_count", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "title": "Celery: active worker processes per queue", + "type": "timeseries", + "description": "Number of worker processes the broker sees, per queue. Previously used celery_worker_tasks_active, which the exporter does not provide while worker task events are off." + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 7, + "targets": [ + { + "expr": "rate(otelcol_receiver_accepted_metric_points_total[5m])", + "legendFormat": "accepted", + "refId": "A" + } + ], + "title": "OTel collector: received metric points/s (empty until SDK instrumentation)", + "type": "timeseries", + "description": "Stays empty until the application itself sends OpenTelemetry metrics. The collector is running but receives nothing yet — this is expected, see backlog item C1." + }, + { + "id": 12, + "type": "timeseries", + "title": "VM restarts (boot time per node)", + "description": "Every upward jump is a reboot of that node. The y-axis shows the boot moment itself; a flat line means no restarts. The freeze-reboot of 28 July is directly visible here.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "fieldConfig": { + "defaults": { + "unit": "dateTimeAsIso", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "lineInterpolation": "stepAfter" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "node_boot_time_seconds * 1000", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Inode usage % (per node, /)", + "description": "Percentage of inodes used on the root filesystem. A disk can run 'full' with gigabytes still free: every file costs an inode, and /srv/nfs (the NFS export with thousands of gherkin log files) lives on this filesystem.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 5, + "showPoints": "auto", + "spanNulls": true, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "100 - (node_filesystem_files_free{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_files{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + } + ], + "refresh": "30s", + "tags": [ + "observability", + "ivs-681" + ], + "time": { + "from": "now-3h", + "to": "now" + }, + "timezone": "browser", + "title": "Validation Service — System Health (DEV)", + "uid": "vs-system-health", + "version": 2, + "schemaVersion": 39, + "editable": true +} \ No newline at end of file diff --git a/docker/grafana/dashboards/vs-validation-perf.json b/docker/grafana/dashboards/vs-validation-perf.json new file mode 100644 index 00000000..4dd5891e --- /dev/null +++ b/docker/grafana/dashboards/vs-validation-perf.json @@ -0,0 +1,604 @@ +{ + "id": null, + "uid": "vs-validation-perf", + "title": "Validation Service — Performance & Load", + "tags": [ + "observability", + "performance", + "postgres" + ], + "editable": true, + "schemaVersion": 39, + "version": 2, + "refresh": "5m", + "time": { + "from": "now-90d", + "to": "now" + }, + "timezone": "browser", + "description": "Prestaties en belasting van de Validation Service, rechtstreeks uit de applicatiedatabase (datasource: DEV Postgres, uid devpg). Doorlooptijd = klok van aanmelden tot klaar (inclusief wachten in de wachtrij). Verwerkingstijd = som van de taakduren (alleen echt rekenwerk).", + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Duration per task type over time (daily average)", + "description": "How long does each validation step take on average per day? Based on ifc_validation_task.ended - started. Follows the dashboard time range.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "showPoints": "auto", + "pointSize": 5, + "spanNulls": true, + "fillOpacity": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroupAlias(t.started, '1d'),\n t.type AS metric,\n ROUND(AVG(EXTRACT(EPOCH FROM (t.ended - t.started)))::numeric, 1) AS duration_s\nFROM ifc_validation_task t\nWHERE $__timeFilter(t.started)\n AND t.started IS NOT NULL\n AND t.ended IS NOT NULL\nGROUP BY 1, 2\nORDER BY 1" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Validation requests per day (by final status)", + "description": "Volume: how many files are submitted per day, and how they ended (COMPLETED / FAILED / PENDING / INITIATED). Source: ifc_validation_request.created.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "FAILED" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "COMPLETED" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroupAlias(r.created, '1d'),\n r.status AS metric,\n COUNT(*)::float AS requests\nFROM ifc_validation_request r\nWHERE $__timeFilter(r.created)\nGROUP BY 1, 2\nORDER BY 1" + } + ] + }, + { + "id": 3, + "type": "table", + "title": "Failure rate per task type", + "description": "Which validation step breaks most often? Failure rate = FAILED / (FAILED + COMPLETED) within the selected time range. Tasks with status SKIPPED / N/A / INITIATED are excluded.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "align": "auto", + "filterable": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failure_rate_pct" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "gradient" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "failure_rate_pct", + "desc": true + } + ] + }, + "targets": [ + { + "refId": "A", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n t.type AS task_type,\n COUNT(*) FILTER (WHERE t.status IN ('COMPLETED','FAILED')) AS completed,\n COUNT(*) FILTER (WHERE t.status = 'FAILED') AS failed,\n ROUND(100.0 * COUNT(*) FILTER (WHERE t.status = 'FAILED')\n / NULLIF(COUNT(*) FILTER (WHERE t.status IN ('COMPLETED','FAILED')), 0), 2) AS failure_rate_pct,\n COUNT(*) FILTER (WHERE t.status IN ('PENDING','INITIATED')) AS unfinished\nFROM ifc_validation_task t\nWHERE $__timeFilter(t.created)\nGROUP BY 1\nORDER BY 4 DESC NULLS LAST, 2 DESC" + } + ] + }, + { + "id": 4, + "type": "barchart", + "title": "Processing time vs file size", + "description": "Does it get slower as files get larger? Files are grouped into size buckets; per bucket the average and p95 of processing time (sum of task durations).", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 80, + "axisPlacement": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "viz": true, + "legend": false, + "tooltip": false + } + } + ] + } + ] + }, + "options": { + "xField": "size", + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "groupWidth": 0.7, + "barWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "format": "table", + "rawQuery": true, + "rawSql": "WITH per_request AS (\n SELECT r.id,\n r.size,\n SUM(EXTRACT(EPOCH FROM (t.ended - t.started))) AS processing_time_s\n FROM ifc_validation_request r\n JOIN ifc_validation_task t ON t.request_id = r.id\n WHERE $__timeFilter(r.created)\n AND r.size IS NOT NULL\n AND t.started IS NOT NULL\n AND t.ended IS NOT NULL\n GROUP BY r.id, r.size\n)\nSELECT\n CASE\n WHEN size < 1048576 THEN '1. < 1 MB'\n WHEN size < 5242880 THEN '2. 1-5 MB'\n WHEN size < 20971520 THEN '3. 5-20 MB'\n WHEN size < 52428800 THEN '4. 20-50 MB'\n WHEN size < 209715200 THEN '5. 50-200 MB'\n ELSE '6. > 200 MB'\n END AS size,\n ROUND(AVG(verwerkingstijd_s)::numeric, 0) AS avg_s,\n ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY verwerkingstijd_s)::numeric, 0) AS p95_s,\n COUNT(*) AS count\nFROM per_request\nGROUP BY 1\nORDER BY 1" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Lead time per request (p50 / p95 per day)", + "description": "How long does a user actually wait? Lead time = completed - created of the request, so including queue time. Large outliers mean queueing or re-runs, not slow validation.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroupAlias(r.created, '1d'),\n ROUND(percentile_cont(0.50) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (r.completed - r.created)))::numeric, 1) AS p50_s,\n ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (r.completed - r.created)))::numeric, 1) AS p95_s\nFROM ifc_validation_request r\nWHERE $__timeFilter(r.created)\n AND r.completed IS NOT NULL\nGROUP BY 1\nORDER BY 1" + } + ] + }, + { + "id": 6, + "type": "table", + "title": "20 slowest requests (last 30 days)", + "description": "Fixed 30-day window, independent of the time range above. Sorted by processing time (sum of task durations). Lead time is shown alongside: a large gap between the two means waiting, not working.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "align": "auto", + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mb" + }, + "properties": [ + { + "id": "unit", + "value": "decmbytes" + }, + { + "id": "decimals", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "processing_time_s" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lead_time_s" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "FAILED": { + "color": "red", + "index": 0, + "text": "FAILED" + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "targets": [ + { + "refId": "A", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n r.created AS submitted,\n r.file_name AS file,\n ROUND(r.size / 1048576.0, 1) AS mb,\n r.status,\n ROUND(v.processing_time_s::numeric, 0) AS processing_time_s,\n ROUND(EXTRACT(EPOCH FROM (r.completed - r.created))::numeric, 0) AS lead_time_s,\n v.langzaamste_taak\nFROM ifc_validation_request r\nJOIN (\n SELECT t.request_id,\n SUM(EXTRACT(EPOCH FROM (t.ended - t.started))) AS processing_time_s,\n (ARRAY_AGG(t.type ORDER BY (t.ended - t.started) DESC))[1] AS slowest_task\n FROM ifc_validation_task t\n WHERE t.started IS NOT NULL AND t.ended IS NOT NULL\n GROUP BY t.request_id\n) v ON v.request_id = r.id\nWHERE r.created > NOW() - INTERVAL '30 days'\nORDER BY v.processing_time_s DESC NULLS LAST\nLIMIT 20" + } + ] + }, + { + "id": 7, + "type": "barchart", + "title": "Most expensive gherkin rules (total CPU time, cumulative)", + "description": "From the gherkin logs on the NFS share (201k measurements since Dec 2025), ingested nightly by perf-metrics/gherkin_rule_timings.sh. NOTE: this is CPU time (time.process_time), not wall-clock time, so I/O wait is not included. Logged in PRODUCTION mode only.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 37 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal" + }, + "targets": [ + { + "refId": "A", + "expr": "topk(15, gherkin_rule_cpu_seconds_total)", + "legendFormat": "{{rule}}", + "format": "time_series", + "instant": true + } + ] + }, + { + "id": 8, + "type": "table", + "title": "Rule cost: total, average and longest run", + "description": "Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 37 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "rule", + "mode": "outer" + } + } + ], + "targets": [ + { + "refId": "A", + "expr": "topk(20, gherkin_rule_cpu_seconds_total)", + "format": "table", + "instant": true, + "legendFormat": "totaal_s" + }, + { + "refId": "B", + "expr": "gherkin_rule_cpu_seconds_avg", + "format": "table", + "instant": true, + "legendFormat": "gem_s" + }, + { + "refId": "C", + "expr": "gherkin_rule_cpu_seconds_max", + "format": "table", + "instant": true, + "legendFormat": "max_s" + }, + { + "refId": "D", + "expr": "gherkin_rule_runs_total", + "format": "table", + "instant": true, + "legendFormat": "runs" + } + ] + }, + { + "id": 9, + "type": "table", + "title": "Outcomes per rule (top 15, whole DB)", + "description": "Which rules produce the most outcomes (severity WARNING/ERROR)? Feeds backlog item F4: SWE001 and IFC105 together account for ~1.7M of the ~5M rows.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 46 + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "targets": [ + { + "refId": "A", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE WHEN regel ~ '^[A-Z]{2,5}[0-9]{2,3}$' THEN regel\n ELSE '(SCHEMA check, not a gherkin rule)' END AS rule,\n SUM(warnings)::bigint AS warnings,\n SUM(errors)::bigint AS errors,\n SUM(totaal)::bigint AS total\nFROM (\n SELECT split_part(feature, ' ', 1) AS rule,\n SUM((severity = 3)::int) AS warnings,\n SUM((severity = 4)::int) AS errors,\n COUNT(*) AS total\n FROM ifc_validation_outcome\n WHERE severity >= 3 AND feature IS NOT NULL\n GROUP BY 1\n) sub\nGROUP BY 1\nORDER BY totaal DESC\nLIMIT 15" + } + ] + } + ] +} \ No newline at end of file diff --git a/docker/grafana/grafana.ini b/docker/grafana/grafana.ini index 5370f485..2c18c2c4 100644 --- a/docker/grafana/grafana.ini +++ b/docker/grafana/grafana.ini @@ -49,10 +49,11 @@ provisioning = /etc/grafana/provisioning # The full public facing url you use in browser, used for redirects and emails # If you use reverse proxy and sub path specify full url (with sub path) -;root_url = %(protocol)s://%(domain)s:%(http_port)s/ +# served behind nginx at /grafana/ (same domain as the Django admin) +root_url = %(protocol)s://%(domain)s/grafana/ # Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. -;serve_from_sub_path = false +serve_from_sub_path = true # Log web requests ;router_logging = false @@ -434,7 +435,9 @@ allow_sign_up = true #################################### Anonymous Auth ###################### [auth.anonymous] # enable anonymous access -enabled = true +# Disabled: Grafana is reachable at /grafana/ on the public domain, so anonymous +# access would expose every dashboard to anyone on the internet. +enabled = false # specify organization name that should be used for unauthenticated users org_name = Main Org. diff --git a/docker/grafana/provisioning/dashboards/provider.yaml b/docker/grafana/provisioning/dashboards/provider.yaml new file mode 100644 index 00000000..2cc58bf9 --- /dev/null +++ b/docker/grafana/provisioning/dashboards/provider.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: false diff --git a/docker/grafana/provisioning/dashboards/vs-dashboards.yaml b/docker/grafana/provisioning/dashboards/vs-dashboards.yaml new file mode 100644 index 00000000..5117e775 --- /dev/null +++ b/docker/grafana/provisioning/dashboards/vs-dashboards.yaml @@ -0,0 +1,20 @@ +# Dashboards as code for the Validation Service. +# +# The JSON files live in this repository under docker/grafana/dashboards/ and are +# mounted into the container at /etc/grafana/dashboards-vs. +# Grafana re-reads them every 30 seconds and after a restart, so nothing is lost +# when a container is recreated. + +apiVersion: 1 + +providers: + - name: "vs-dashboards" + orgId: 1 + folder: "Validation Service" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /etc/grafana/dashboards-vs + foldersFromFilesStructure: false diff --git a/docker/grafana/provisioning/datasources/default.yaml b/docker/grafana/provisioning/datasources/default.yaml index be1acd40..2331ab7a 100644 --- a/docker/grafana/provisioning/datasources/default.yaml +++ b/docker/grafana/provisioning/datasources/default.yaml @@ -1,7 +1,9 @@ -apiVersion: 1 - -datasources: - - name: Open-Telemetry-Example - type: prometheus - url: http://prometheus:9090 - editable: true \ No newline at end of file +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/docker/grafana/provisioning/datasources/postgres.yaml b/docker/grafana/provisioning/datasources/postgres.yaml new file mode 100644 index 00000000..81a1dbd8 --- /dev/null +++ b/docker/grafana/provisioning/datasources/postgres.yaml @@ -0,0 +1,31 @@ +# Provisioned PostgreSQL datasource for the Azure managed Postgres. + +apiVersion: 1 + +# Remove the datasource this one replaces. Earlier deployments provisioned the same +# uid under the name "DEV Postgres"; without this, Grafana tries to insert a second +# datasource with an existing uid and refuses to start. +deleteDatasources: + - name: DEV Postgres + orgId: 1 + +datasources: + - name: Validation Service DB + uid: devpg + type: grafana-postgresql-datasource + access: proxy + url: $POSTGRES_HOST:$POSTGRES_PORT + user: $POSTGRES_USER + database: $POSTGRES_NAME + isDefault: false + editable: false + jsonData: + sslmode: require + postgresVersion: 1600 + timescaledb: false + maxOpenConns: 2 + maxIdleConns: 1 + maxIdleConnsAuto: false + connMaxLifetime: 14400 + secureJsonData: + password: $POSTGRES_PASSWORD diff --git a/docker/otel/otel-collector-config.yaml b/docker/otel/otel-collector-config.yaml index f4853222..fb046cf2 100644 --- a/docker/otel/otel-collector-config.yaml +++ b/docker/otel/otel-collector-config.yaml @@ -1,45 +1,65 @@ -# see https://opentelemetry.io/docs/collector/configuration/ - -# receivers configure how telemetry data gets into the collector -receivers: - otlp: - protocols: - grpc: - http: - -# processors specify what happens with the received telemetry data -processors: - batch: - send_batch_size: 1000 - timeout: 10s - -# exporters configure how to send processed data to backend(s) -exporters: - # azuremonitorexporter: - # endpoint: azure_monitor_otlp:4317 # TODO - prometheusremotewrite: - endpoint: 0.0.0.0:8889 - logging: - -# optional components that expand the capabilities of the collector -extensions: - health_check: - pprof: - zpages: - -# pipelines glue the receivers, processors, and exporters together -service: - extensions: [health_check, pprof, zpages] - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [prometheus, logging] - metrics: - receivers: [otlp] - processors: [batch] - exporters: [prometheus, logging] - logs: - receivers: [otlp] - processors: [batch] - exporters: [prometheus, logging] \ No newline at end of file +# see https://opentelemetry.io/docs/collector/configuration/ + +# receivers configure how telemetry data gets into the collector +receivers: + otlp: + protocols: + # Bind to 0.0.0.0 explicitly. Since collector version 0.104 the default is + # localhost-only, which makes the receiver unreachable from other containers + # on the overlay network. + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +# processors specify what happens with the received telemetry data +processors: + batch: + send_batch_size: 1000 + timeout: 10s + +# exporters configure how to send processed data to backend(s) +exporters: + # azuremonitorexporter: + # endpoint: azure_monitor_otlp:4317 # TODO + # Pull-based exporter: Prometheus scrapes these metrics from the collector on :8889. + prometheus: + endpoint: 0.0.0.0:8889 + debug: + +# optional components that expand the capabilities of the collector +extensions: + health_check: + pprof: + zpages: + +# pipelines glue the receivers, processors, and exporters together +service: + extensions: [health_check, pprof, zpages] + telemetry: + metrics: + # Expose the collector's own metrics on :8888 so Prometheus can scrape them + # (the default binds to localhost only). + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + # The prometheus exporter handles metrics only. Traces and logs therefore go to + # the debug exporter, which writes them to the collector's own log. Every + # pipeline needs at least one exporter or the collector refuses to start. + # todo gh: Replace debug with a real trace backend once chosen. + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus, debug] + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug] diff --git a/docker/prometheus/prometheus.yaml b/docker/prometheus/prometheus.yaml index 1f25d465..950fb19a 100644 --- a/docker/prometheus/prometheus.yaml +++ b/docker/prometheus/prometheus.yaml @@ -1,6 +1,23 @@ -scrape_configs: - - job_name: "otel-collector" - scrape_interval: 10s - static_configs: - - targets: ["otel-collector:8889"] - - targets: ["otel-collector:8888"] \ No newline at end of file +global: + scrape_interval: 15s + +scrape_configs: + # The OpenTelemetry collector exposes two endpoints: its own health/telemetry on + # :8888, and any application metrics it received over OTLP on :8889. + - job_name: "otel-collector" + scrape_interval: 10s + static_configs: + - targets: ["otel_col:8888", "otel_col:8889"] + + # node_exporter runs in swarm global mode (one task per node). tasks. is + # the swarm DNS name that resolves to every task, so dns_sd finds all nodes + # automatically, including nodes added later. + - job_name: "node" + dns_sd_configs: + - names: ["tasks.node_exporter"] + type: A + port: 9100 + + - job_name: "celery" + static_configs: + - targets: ["celery_exporter:9808"]