Carsten Rehfeld
117 lines
4.5 KiB
YAML
117 lines
4.5 KiB
YAML
# Promtail config: Bunny.net CDN access logs → Loki
|
|
#
|
|
# Deploy on the same VPS as the APIX registry (or any host reachable from
|
|
# the internet on port 5514). Bunny.net streams one TCP Syslog line per
|
|
# request — including CDN cache hits — giving live telemetry that the
|
|
# origin Prometheus metrics alone cannot provide.
|
|
#
|
|
# Setup:
|
|
# 1. Install Promtail: https://grafana.com/docs/loki/latest/clients/promtail/installation/
|
|
# 2. Copy this file to /etc/promtail/cdn-logs.yaml
|
|
# 3. Fill in LOKI_PUSH_URL, LOKI_USERNAME, LOKI_PASSWORD below
|
|
# 4. Open TCP 5514 in the VPS firewall
|
|
# 5. systemctl enable --now promtail
|
|
# 6. Run setup-bunnynet.sh with SYSLOG_HOST=<this-vps-ip>
|
|
#
|
|
# Verify stream is working:
|
|
# curl -s https://api-index.org/services?capability=nlp
|
|
# journalctl -u promtail -f # should show the log line within 1-2 seconds
|
|
#
|
|
# Grafana dashboard queries (LogQL):
|
|
# Capability hit rate (live):
|
|
# count_over_time({job="apix-cdn"} | label_format capability="capability" [1m])
|
|
# Cache ratio:
|
|
# sum by (cache_status) (count_over_time({job="apix-cdn"}[5m]))
|
|
# Zero-result detection (requires apix.search.result_count from Prometheus):
|
|
# use a mixed datasource panel combining Loki counts with Prometheus summaries
|
|
|
|
server:
|
|
http_listen_port: 9080
|
|
grpc_listen_port: 0
|
|
log_level: info
|
|
|
|
clients:
|
|
- url: https://LOKI_PUSH_URL/loki/api/v1/push # e.g. logs-prod-xxx.grafana.net/loki/api/v1/push
|
|
basic_auth:
|
|
username: "LOKI_USERNAME" # Grafana Cloud: stack user ID (numeric)
|
|
password: "LOKI_PASSWORD" # Grafana Cloud: service account token (logs:write scope)
|
|
|
|
positions:
|
|
filename: /tmp/promtail-cdn-positions.yaml
|
|
|
|
scrape_configs:
|
|
- job_name: apix-cdn-logs
|
|
syslog:
|
|
listen_address: 0.0.0.0:5514
|
|
label_structured_data: false
|
|
labels:
|
|
job: apix-cdn
|
|
component: bunnynet
|
|
|
|
relabel_configs:
|
|
# Bunny.net includes the PoP hostname in the syslog HOSTNAME field
|
|
# e.g. "pa.b-cdn.net" (Paris), "sg.b-cdn.net" (Singapore)
|
|
# This is how PoPs are visible even on cache hits that never reach origin.
|
|
- source_labels: [__syslog_message_hostname]
|
|
target_label: cdn_pop
|
|
|
|
pipeline_stages:
|
|
# ── Parse Apache Combined Log Format + Bunny.net cache status suffix ──
|
|
# Line: 1.2.3.x - - [12/May/2026:10:00:00 +0000] "GET /services?cap=nlp HTTP/1.1" 200 1234 "-" "Agent/1.0" HIT
|
|
- regex:
|
|
expression: '"(?P<http_method>[A-Z]+) (?P<full_path>[^" ]+) HTTP/[^"]*" (?P<http_status>\d+) \d+ "[^"]*" "[^"]*" (?P<cache_status>\S+)$'
|
|
|
|
- labels:
|
|
http_method:
|
|
http_status:
|
|
cache_status: # HIT | MISS | BYPASS — primary cache ratio signal
|
|
|
|
# ── Split path from query string ──────────────────────────────────────
|
|
- regex:
|
|
source: full_path
|
|
expression: '^(?P<req_path>[^?]+)(\?(?P<query_string>.+))?$'
|
|
|
|
- labels:
|
|
req_path: # /services /devices /devices/{id}/replacements
|
|
|
|
# ── Extract search parameters ─────────────────────────────────────────
|
|
# capability is the primary analytics dimension
|
|
- regex:
|
|
source: query_string
|
|
expression: '(?:^|&)capability=(?P<capability>[^&]*)'
|
|
|
|
- labels:
|
|
capability: # nlp | translation | speech-to-text | … | (empty = nav doc)
|
|
|
|
- regex:
|
|
source: query_string
|
|
expression: '(?:^|&)deviceClass=(?P<device_class>[^&]*)'
|
|
|
|
- labels:
|
|
device_class: # sensor | actuator | gateway | …
|
|
|
|
- regex:
|
|
source: query_string
|
|
expression: '(?:^|&)protocol=(?P<protocol>[^&]*)'
|
|
|
|
- labels:
|
|
protocol: # MQTT | AMQP | HTTP | …
|
|
|
|
# stage only appears explicitly when non-PRODUCTION (canonical form omits it)
|
|
- regex:
|
|
source: query_string
|
|
expression: '(?:^|&)stage=(?P<stage>[^&]*)'
|
|
|
|
- labels:
|
|
stage: # STAGING | DEVELOPMENT — absent means PRODUCTION
|
|
|
|
# ── Drop Quarkus internal paths — never cache, no analytics value ─────
|
|
- drop:
|
|
source: req_path
|
|
expression: '^/q/'
|
|
|
|
# ── Timestamp from log line (keeps Loki time accurate) ────────────────
|
|
- timestamp:
|
|
source: __syslog_message_timestamp
|
|
format: RFC3339
|