diff --git a/.gitignore b/.gitignore index ada1ac8..7ba8b66 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ -# Script run logs +# Script run logs and runtime artifacts logs/ +.logs/ +.pids/ +C*Temp*.json # flatten-maven-plugin resolves ${revision} into installed POMs .flattened-pom.xml diff --git a/scripts/deploy-portal.ps1 b/scripts/deploy-portal.ps1 new file mode 100644 index 0000000..7d489c6 --- /dev/null +++ b/scripts/deploy-portal.ps1 @@ -0,0 +1,23 @@ +$ErrorActionPreference = "Stop" +$VPS = "deploy@204.168.156.179" +$MVN = "C:\Users\Anwender\IdeaProjects\Claude\bot-service-index\apix-mvp" + +Set-Location $MVN + +Write-Host "Copying web content..." +Copy-Item "$MVN\..\web\api-index.org\index.html" ` + "$MVN\apix-portal\src\main\resources\META-INF\resources\index.html" -Force + +Write-Host "Building portal..." +mvn clean package -DskipTests -pl apix-portal -am -q + +Write-Host "Transferring to VPS..." +scp "$MVN\apix-portal\target\quarkus-app\app\apix-portal-1.0-SNAPSHOT.jar" ` + "${VPS}:/opt/apix/apix-portal/target/quarkus-app/app/apix-portal-1.0-SNAPSHOT.jar" +scp "$MVN\apix-portal\target\quarkus-app\quarkus\quarkus-application.dat" ` + "${VPS}:/opt/apix/apix-portal/target/quarkus-app/quarkus/quarkus-application.dat" + +Write-Host "Rebuilding image and restarting portal..." +ssh $VPS "cd /opt/apix && docker build -f infra/Dockerfile.portal -t apix-portal:latest . -q && docker compose -f infra/docker-compose.yml --env-file .env up -d portal" + +Write-Host "Done. https://www.api-index.org/" diff --git a/scripts/deploy-registry.ps1 b/scripts/deploy-registry.ps1 new file mode 100644 index 0000000..ee7da85 --- /dev/null +++ b/scripts/deploy-registry.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" +$VPS = "deploy@204.168.156.179" +$MVN = "C:\Users\Anwender\IdeaProjects\Claude\bot-service-index\apix-mvp" +$QUARKUS_APP = "$MVN\apix-registry\target\quarkus-app" +$VPS_DEST = "/opt/apix/apix-registry/target/quarkus-app" + +Set-Location $MVN + +Write-Host "Building registry..." +mvn clean package -DskipTests -pl apix-registry -am -q + +Write-Host "Syncing quarkus-app to VPS (rsync — only changed files)..." +# cwrsync requires /cygdrive/c/... path format +$rsyncSrc = $QUARKUS_APP -replace '^([A-Za-z]):\\', '/cygdrive/$1/' -replace '\\', '/' +rsync -az --delete ` + "${rsyncSrc}/" ` + "${VPS}:${VPS_DEST}/" + +Write-Host "Rebuilding Docker image on VPS..." +ssh $VPS "cd /opt/apix && docker build -f infra/Dockerfile.registry -t apix-registry:latest . -q" + +Write-Host "Rolling restart: registry-a..." +ssh $VPS "cd /opt/apix && docker compose -f infra/docker-compose.yml --env-file .env up -d --no-deps registry-a" + +Write-Host "Waiting for registry-a to become healthy..." +$elapsed = 0 +do { + Start-Sleep -Seconds 5; $elapsed += 5 + $status = ssh $VPS "docker inspect infra-registry-a-1 --format '{{.State.Health.Status}}' 2>/dev/null || echo missing" + Write-Host " registry-a: $status ($elapsed s)" +} while ($status -ne "healthy" -and $elapsed -lt 120) + +if ($status -ne "healthy") { + Write-Host "ERROR: registry-a did not become healthy — aborting, registry-b untouched" + ssh $VPS "docker logs infra-registry-a-1 --tail 20 2>&1" + exit 1 +} + +Write-Host "Restarting registry-b..." +ssh $VPS "cd /opt/apix && docker compose -f infra/docker-compose.yml --env-file .env up -d --no-deps registry-b" + +Write-Host "Waiting for registry-b..." +$elapsed = 0 +do { + Start-Sleep -Seconds 5; $elapsed += 5 + $statusB = ssh $VPS "docker inspect infra-registry-b-1 --format '{{.State.Health.Status}}' 2>/dev/null || echo missing" + Write-Host " registry-b: $statusB ($elapsed s)" +} while ($statusB -ne "healthy" -and $elapsed -lt 120) + +Write-Host "registry-a: $status registry-b: $statusB" +Write-Host "Done. https://api-index.org/" diff --git a/scripts/generate-env.sh b/scripts/generate-env.sh new file mode 100644 index 0000000..f77d5c2 --- /dev/null +++ b/scripts/generate-env.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +cat > /opt/apix/.env << EOF +APIX_DB_USER=apix +APIX_DB_PASSWORD=$(openssl rand -base64 32) +APIX_DB_NAME=apix +APIX_DB_PORT=5432 +APIX_API_KEY=$(openssl rand -hex 32) +APIX_REGISTRY_BASE_URL=https://api-index.org +APIX_REGISTRY_NAME=APIX Registry +GLEIF_API_URL=https://api.gleif.org/api/v1 +OPENCORPORATES_API_KEY= +APIX_MAIL_SIGNING_PRIVATE_KEY= +APIX_MAIL_SIGNING_PUBLIC_KEY= +APIX_MAIL_SIGNING_KID=2026-05 +SPIDER_INTERVAL_MINUTES=15 +GRAFANA_ADMIN_PASSWORD=$(openssl rand -base64 16) +GRAFANA_ROOT_URL=http://localhost:3000 +LOG_LEVEL=INFO +EOF + +chmod 600 /opt/apix/.env +echo "Generated /opt/apix/.env:" +cat /opt/apix/.env diff --git a/scripts/grafana-demo-dashboard.json b/scripts/grafana-demo-dashboard.json new file mode 100644 index 0000000..abfc6a3 --- /dev/null +++ b/scripts/grafana-demo-dashboard.json @@ -0,0 +1,603 @@ +{ + "__inputs": [ + { + "name": "DS_LOKI", + "label": "Loki", + "description": "Loki receiving Bunny.net CDN access logs via Promtail (promtail-cdn-logs.yaml). Shows ALL requests including CDN cache hits.", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + }, + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus scraping the APIX registry /q/metrics. Shows only cache misses that reached the origin.", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, + { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, + { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, + { "type": "panel", "id": "gauge", "name": "Gauge", "version": "" }, + { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" }, + { "type": "panel", "id": "piechart", "name": "Pie chart", "version": "" }, + { "type": "panel", "id": "barchart", "name": "Bar chart", "version": "" }, + { "type": "datasource", "id": "loki", "name": "Loki", "version": "" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "" } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Live APIX Registry traffic — OpenClaw community demo. Loki panels cover ALL requests (CDN cache hits + misses) via Bunny.net syslog forwarding. Prometheus panels cover origin cache misses only. Refresh every 5s.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "refresh": "5s", + "schemaVersion": 38, + "tags": ["apix", "demo", "openclaw"], + "time": { "from": "now-15m", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "APIX Registry — OpenClaw Demo", + "uid": "apix-demo-v1", + "version": 1, + + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Loki datasource", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "refresh": 1, + "type": "datasource" + }, + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + } + ] + }, + + "panels": [ + + { + "id": 1, + "type": "stat", + "title": "CDN Searches", + "description": "Total /services + /devices requests in the selected time range, including CDN cache hits.", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 3 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "#1f60c4", "value": null }, + { "color": "#37872d", "value": 500 }, + { "color": "#8f3bb8", "value": 10000 } + ] + }, + "unit": "short", + "noValue": "0" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "sum(count_over_time({job=\"apix-cdn\", req_path=~\"/services.*|/devices.*\"}[$__range]))", + "instant": true, + "queryType": "instant", + "legendFormat": "" + } + ] + }, + + { + "id": 2, + "type": "gauge", + "title": "Cache Hit Ratio", + "description": "Percentage of CDN requests served from edge cache without hitting the origin VPS.", + "gridPos": { "x": 6, "y": 0, "w": 6, "h": 3 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "minVizWidth": 75, + "minVizHeight": 75 + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "#c4162a", "value": null }, + { "color": "#ff9830", "value": 50 }, + { "color": "#37872d", "value": 80 } + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "noValue": "—" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "sum(count_over_time({job=\"apix-cdn\", cache_status=\"HIT\"}[$__range])) / sum(count_over_time({job=\"apix-cdn\"}[$__range])) * 100", + "instant": true, + "queryType": "instant", + "legendFormat": "" + } + ] + }, + + { + "id": 3, + "type": "stat", + "title": "Active PoPs", + "description": "Number of distinct Bunny.net edge PoPs that served traffic in the selected time range.", + "gridPos": { "x": 12, "y": 0, "w": 6, "h": 3 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "#1f60c4", "value": null }, + { "color": "#37872d", "value": 3 }, + { "color": "#8f3bb8", "value": 8 } + ] + }, + "unit": "short", + "noValue": "0" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "count(sum by (cdn_pop) (count_over_time({job=\"apix-cdn\"}[$__range])))", + "instant": true, + "queryType": "instant", + "legendFormat": "" + } + ] + }, + + { + "id": 4, + "type": "stat", + "title": "Searches / sec", + "description": "Current CDN search request rate across all edge PoPs.", + "gridPos": { "x": 18, "y": 0, "w": 6, "h": 3 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "#1f60c4", "value": null }, + { "color": "#37872d", "value": 1 }, + { "color": "#f2cc0c", "value": 10 }, + { "color": "#8f3bb8", "value": 100 } + ] + }, + "unit": "reqps", + "decimals": 2, + "noValue": "0" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "sum(rate({job=\"apix-cdn\", req_path=~\"/services.*|/devices.*\"}[$__rate_interval]))", + "instant": false, + "queryType": "range", + "legendFormat": "" + } + ] + }, + + { + "id": 10, + "type": "row", + "title": "Capability Traffic — Real-Time", + "collapsed": false, + "gridPos": { "x": 0, "y": 3, "w": 24, "h": 1 }, + "panels": [] + }, + + { + "id": 5, + "type": "timeseries", + "title": "Requests per Second by Capability", + "description": "Live stream of capability search rates across all CDN PoPs. Each line = one capability being queried. This includes CDN cache hits — the spike you see is the real community traffic.", + "gridPos": { "x": 0, "y": 4, "w": 24, "h": 8 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["lastNotNull", "max", "sum"] + } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": false + }, + "unit": "reqps" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "sum by (capability) (rate({job=\"apix-cdn\", capability!=\"\"}[$__rate_interval]))", + "instant": false, + "queryType": "range", + "legendFormat": "{{capability}}" + } + ] + }, + + { + "id": 20, + "type": "row", + "title": "Distribution", + "collapsed": false, + "gridPos": { "x": 0, "y": 12, "w": 24, "h": 1 }, + "panels": [] + }, + + { + "id": 6, + "type": "bargauge", + "title": "Top 10 Capabilities", + "description": "Most queried capabilities in the time range (all CDN traffic). These are the API categories the OpenClaw community is actively using.", + "gridPos": { "x": 0, "y": 13, "w": 15, "h": 8 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "horizontal", + "displayMode": "gradient", + "valueMode": "color", + "showUnfilled": true, + "minVizWidth": 8, + "minVizHeight": 16, + "namePlacement": "left", + "legendDisplayMode": "hidden" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "continuous-BlPu" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue", "value": null }] + }, + "unit": "short" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "topk(10, sum by (capability) (count_over_time({job=\"apix-cdn\", capability!=\"\"}[$__range])))", + "instant": true, + "queryType": "instant", + "legendFormat": "{{capability}}" + } + ] + }, + + { + "id": 7, + "type": "piechart", + "title": "Cache Status", + "description": "HIT = served from CDN edge, no origin load. MISS = edge cache cold, forwarded to origin. BYPASS = edge rule prevented caching (/q/* health paths).", + "gridPos": { "x": 15, "y": 13, "w": 9, "h": 8 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "pieType": "donut", + "displayLabels": ["name", "percent"], + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["sum"] + } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed" }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "HIT" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#37872d" } }] + }, + { + "matcher": { "id": "byName", "options": "MISS" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#ff9830" } }] + }, + { + "matcher": { "id": "byName", "options": "BYPASS" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#6e7279" } }] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "sum by (cache_status) (count_over_time({job=\"apix-cdn\"}[$__range]))", + "instant": true, + "queryType": "instant", + "legendFormat": "{{cache_status}}" + } + ] + }, + + { + "id": 30, + "type": "row", + "title": "Geographic Distribution", + "collapsed": false, + "gridPos": { "x": 0, "y": 21, "w": 24, "h": 1 }, + "panels": [] + }, + + { + "id": 8, + "type": "barchart", + "title": "Requests by CDN PoP", + "description": "Which Bunny.net edge PoPs served traffic. PoP names come from the syslog HOSTNAME field (e.g. sg.b-cdn.net = Singapore, hk.b-cdn.net = Hong Kong). Asia traffic is visible here even though the origin VPS is in Europe.", + "gridPos": { "x": 0, "y": 22, "w": 16, "h": 7 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "orientation": "horizontal", + "xTickLabelRotation": 0, + "xTickLabelMaxLength": 30, + "groupWidth": 0.7, + "barWidth": 0.97, + "fillOpacity": 80, + "gradientMode": "none", + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "hidden", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "continuous-BlYlRd" }, + "custom": { "fillOpacity": 80 }, + "unit": "short" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "topk(15, sum by (cdn_pop) (count_over_time({job=\"apix-cdn\"}[$__range])))", + "instant": true, + "queryType": "instant", + "legendFormat": "{{cdn_pop}}" + } + ] + }, + + { + "id": 9, + "type": "piechart", + "title": "/services vs /devices", + "description": "Split of AI agent API searches (/services) vs IoT device discovery (/devices). As the OpenClaw community grows, /services should dominate initially, with /devices rising as embedded systems adopt APIX.", + "gridPos": { "x": 16, "y": 22, "w": 8, "h": 7 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "pieType": "pie", + "displayLabels": ["name", "percent"], + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } + }, + "fieldConfig": { + "defaults": { "unit": "short" }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "/services" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#1f60c4" } }] + }, + { + "matcher": { "id": "byName", "options": "/devices" }, + "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "#37872d" } }] + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "editorMode": "code", + "expr": "sum by (req_path) (count_over_time({job=\"apix-cdn\", req_path=~\"/services|/devices\"}[$__range]))", + "instant": true, + "queryType": "instant", + "legendFormat": "{{req_path}}" + } + ] + }, + + { + "id": 40, + "type": "row", + "title": "Origin Metrics (cache misses only — Prometheus)", + "collapsed": false, + "gridPos": { "x": 0, "y": 29, "w": 24, "h": 1 }, + "panels": [] + }, + + { + "id": 11, + "type": "timeseries", + "title": "Avg Results per Search (origin)", + "description": "Average result set size per capability search, measured at origin. Low values mean a capability is registered but has few services. Zero means no services are registered for that capability yet — demand without supply.", + "gridPos": { "x": 0, "y": 30, "w": 12, "h": 7 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["lastNotNull", "mean"] + } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "lineWidth": 2, + "fillOpacity": 5, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short", + "decimals": 1 + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "rate(apix_search_result_count_sum{resource=\"services\"}[$__rate_interval]) / rate(apix_search_result_count_count{resource=\"services\"}[$__rate_interval])", + "legendFormat": "services", + "instant": false, + "range": true + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "rate(apix_search_result_count_sum{resource=\"devices\"}[$__rate_interval]) / rate(apix_search_result_count_count{resource=\"devices\"}[$__rate_interval])", + "legendFormat": "devices", + "instant": false, + "range": true + } + ] + }, + + { + "id": 12, + "type": "timeseries", + "title": "Origin Search Rate by Capability (cache misses)", + "description": "Prometheus counters at origin — only incremented on CDN cache misses. The ratio between this panel and the Loki 'Requests/sec' panel above shows the CDN offload factor: if origin sees 1 req/s and Loki shows 50 req/s, the CDN is absorbing 98% of traffic.", + "gridPos": { "x": 12, "y": 30, "w": 12, "h": 7 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["lastNotNull", "max"] + } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "lineWidth": 1, + "fillOpacity": 4, + "showPoints": "never", + "spanNulls": true, + "lineStyle": { "fill": "dash", "dash": [4, 4] } + }, + "unit": "reqps", + "decimals": 3 + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (capability) (rate(apix_search_services_total[$__rate_interval]))", + "legendFormat": "{{capability}}", + "instant": false, + "range": true + } + ] + } + + ] +} diff --git a/scripts/promtail-cdn-logs.yaml b/scripts/promtail-cdn-logs.yaml new file mode 100644 index 0000000..a576aee --- /dev/null +++ b/scripts/promtail-cdn-logs.yaml @@ -0,0 +1,116 @@ +# Promtail config: Bunny.net CDN access logs → Loki +# +# Deploy on the same VPS as the APIX registry (or any host reachable from +# the internet on port 5514). Bunny.net streams one TCP Syslog line per +# request — including CDN cache hits — giving live telemetry that the +# origin Prometheus metrics alone cannot provide. +# +# Setup: +# 1. Install Promtail: https://grafana.com/docs/loki/latest/clients/promtail/installation/ +# 2. Copy this file to /etc/promtail/cdn-logs.yaml +# 3. Fill in LOKI_PUSH_URL, LOKI_USERNAME, LOKI_PASSWORD below +# 4. Open TCP 5514 in the VPS firewall +# 5. systemctl enable --now promtail +# 6. Run setup-bunnynet.sh with SYSLOG_HOST= +# +# Verify stream is working: +# curl -s https://api-index.org/services?capability=nlp +# journalctl -u promtail -f # should show the log line within 1-2 seconds +# +# Grafana dashboard queries (LogQL): +# Capability hit rate (live): +# count_over_time({job="apix-cdn"} | label_format capability="capability" [1m]) +# Cache ratio: +# sum by (cache_status) (count_over_time({job="apix-cdn"}[5m])) +# Zero-result detection (requires apix.search.result_count from Prometheus): +# use a mixed datasource panel combining Loki counts with Prometheus summaries + +server: + http_listen_port: 9080 + grpc_listen_port: 0 + log_level: info + +clients: + - url: https://LOKI_PUSH_URL/loki/api/v1/push # e.g. logs-prod-xxx.grafana.net/loki/api/v1/push + basic_auth: + username: "LOKI_USERNAME" # Grafana Cloud: stack user ID (numeric) + password: "LOKI_PASSWORD" # Grafana Cloud: service account token (logs:write scope) + +positions: + filename: /tmp/promtail-cdn-positions.yaml + +scrape_configs: + - job_name: apix-cdn-logs + syslog: + listen_address: 0.0.0.0:5514 + label_structured_data: false + labels: + job: apix-cdn + component: bunnynet + + relabel_configs: + # Bunny.net includes the PoP hostname in the syslog HOSTNAME field + # e.g. "pa.b-cdn.net" (Paris), "sg.b-cdn.net" (Singapore) + # This is how PoPs are visible even on cache hits that never reach origin. + - source_labels: [__syslog_message_hostname] + target_label: cdn_pop + + pipeline_stages: + # ── Parse Apache Combined Log Format + Bunny.net cache status suffix ── + # Line: 1.2.3.x - - [12/May/2026:10:00:00 +0000] "GET /services?cap=nlp HTTP/1.1" 200 1234 "-" "Agent/1.0" HIT + - regex: + expression: '"(?P[A-Z]+) (?P[^" ]+) HTTP/[^"]*" (?P\d+) \d+ "[^"]*" "[^"]*" (?P\S+)$' + + - labels: + http_method: + http_status: + cache_status: # HIT | MISS | BYPASS — primary cache ratio signal + + # ── Split path from query string ────────────────────────────────────── + - regex: + source: full_path + expression: '^(?P[^?]+)(\?(?P.+))?$' + + - labels: + req_path: # /services /devices /devices/{id}/replacements + + # ── Extract search parameters ───────────────────────────────────────── + # capability is the primary analytics dimension + - regex: + source: query_string + expression: '(?:^|&)capability=(?P[^&]*)' + + - labels: + capability: # nlp | translation | speech-to-text | … | (empty = nav doc) + + - regex: + source: query_string + expression: '(?:^|&)deviceClass=(?P[^&]*)' + + - labels: + device_class: # sensor | actuator | gateway | … + + - regex: + source: query_string + expression: '(?:^|&)protocol=(?P[^&]*)' + + - labels: + protocol: # MQTT | AMQP | HTTP | … + + # stage only appears explicitly when non-PRODUCTION (canonical form omits it) + - regex: + source: query_string + expression: '(?:^|&)stage=(?P[^&]*)' + + - labels: + stage: # STAGING | DEVELOPMENT — absent means PRODUCTION + + # ── Drop Quarkus internal paths — never cache, no analytics value ───── + - drop: + source: req_path + expression: '^/q/' + + # ── Timestamp from log line (keeps Loki time accurate) ──────────────── + - timestamp: + source: __syslog_message_timestamp + format: RFC3339 diff --git a/scripts/query-report.sh b/scripts/query-report.sh new file mode 100644 index 0000000..820963d --- /dev/null +++ b/scripts/query-report.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Download Bunny.net access logs and produce a capability frequency report. +# +# This is the only way to see CDN-cached request frequency — the origin +# Prometheus metrics only see cache misses. A weekly cron run pushing the +# output to Grafana covers >99% of analysis needs without real-time overhead. +# +# What this script answers: +# - Which capabilities are queried most often (all requests, including hits)? +# - Which device class / protocol combinations are active? +# - What is the cache hit ratio per endpoint? +# - Which full query string combinations are most common? +# +# Usage: +# BUNNYNET_API_KEY=your-key PULL_ZONE_ID=12345 ./scripts/query-report.sh +# BUNNYNET_API_KEY=your-key PULL_ZONE_ID=12345 DAYS=30 ./scripts/query-report.sh +# +# Output: +# Human-readable report to stdout. +# With PROMETHEUS_PUSH_URL set, also pushes counters to a Prometheus +# Pushgateway (e.g. your dedicated analytics Grafana stack) so the weekly +# numbers become a time-series rather than a point-in-time dump. +# +# PROMETHEUS_PUSH_URL=https://pushgateway.your-grafana.example/metrics/job/apix-cdn-report \ +# BUNNYNET_API_KEY=your-key PULL_ZONE_ID=12345 ./scripts/query-report.sh +# +# Prerequisites: curl, gzip, awk (any POSIX awk) + +set -euo pipefail + +BUNNYNET_API_KEY="${BUNNYNET_API_KEY:?BUNNYNET_API_KEY is required}" +PULL_ZONE_ID="${PULL_ZONE_ID:-$(cat .bunnynet-pull-zone-id 2>/dev/null || echo '')}" +PULL_ZONE_ID="${PULL_ZONE_ID:?PULL_ZONE_ID is required (or run setup-bunnynet.sh first)}" +DAYS="${DAYS:-7}" +PROMETHEUS_PUSH_URL="${PROMETHEUS_PUSH_URL:-}" +LOGGING_BASE="https://logging.bunnycdn.com" + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +touch "${WORK}/combined.log" + +# ── 1. Download logs ────────────────────────────────────────────────────────── + +echo "==> Downloading Bunny.net access logs — pull zone ${PULL_ZONE_ID}, last ${DAYS} days ..." + +for i in $(seq 1 "$DAYS"); do + # Support both GNU date (Linux) and BSD date (macOS) + if date --version >/dev/null 2>&1; then + DATE=$(date -d "-${i} days" +%Y/%m/%d) + else + DATE=$(date -v "-${i}d" +%Y/%m/%d) + fi + Y=$(echo "$DATE" | cut -d/ -f1) + M=$(echo "$DATE" | cut -d/ -f2) + D=$(echo "$DATE" | cut -d/ -f3) + + GZ="${WORK}/${Y}-${M}-${D}.gz" + STATUS=$(curl -s -o "$GZ" -w "%{http_code}" \ + -H "AccessKey: ${BUNNYNET_API_KEY}" \ + "${LOGGING_BASE}/${Y}/${M}/${D}/${PULL_ZONE_ID}/") + + if [ "$STATUS" = "200" ] && [ -s "$GZ" ]; then + gunzip -c "$GZ" >> "${WORK}/combined.log" + printf " %s-%s-%s OK\n" "$Y" "$M" "$D" + else + printf " %s-%s-%s no data (HTTP %s)\n" "$Y" "$M" "$D" "$STATUS" + fi +done + +if [ ! -s "${WORK}/combined.log" ]; then + echo "" + echo "No log data found for the requested period." + exit 0 +fi + +# ── 2. Isolate search requests ──────────────────────────────────────────────── + +grep -E '"GET /services\?' "${WORK}/combined.log" > "${WORK}/svc.log" || true +grep -E '"GET /devices\?' "${WORK}/combined.log" > "${WORK}/dev.log" || true +grep -E '"GET /devices\?|"GET /services\?' "${WORK}/combined.log" > "${WORK}/search.log" || true +grep -E '"GET /[^"?]+/replacements\?' "${WORK}/combined.log" > "${WORK}/repl.log" || true + +TOTAL=$(wc -l < "${WORK}/combined.log") +SEARCH=$(wc -l < "${WORK}/search.log") + +echo "" +printf "Total log entries : %d\n" "$TOTAL" +printf "Search requests : %d (GET /services + GET /devices)\n" "$SEARCH" +echo "" + +# ── Helper: extract query string from a log line ────────────────────────────── +# Input line: 1.2.3.x - - [date] "GET /services?capability=nlp HTTP/1.1" 200 … HIT +# Output: capability=nlp + +extract_qs() { + awk '{ + if (match($0, /"GET [^?]+"?\?([^" ]+) HTTP/, arr)) print arr[1] + }' "$1" +} + +extract_param() { + local param="$1" file="$2" + extract_qs "$file" | tr "&" "\n" | grep "^${param}=" | cut -d= -f2- +} + +# ── 3. Cache hit ratio ──────────────────────────────────────────────────────── + +echo "=== Cache hit ratio (search endpoints) ===================================" +HITS=$(awk '$NF=="HIT"' "${WORK}/search.log" | wc -l || echo 0) +MISS=$(awk '$NF=="MISS"' "${WORK}/search.log" | wc -l || echo 0) +BYPS=$(awk '$NF=="BYPASS"' "${WORK}/search.log" | wc -l || echo 0) +printf " HIT : %d\n" "$HITS" +printf " MISS : %d\n" "$MISS" +printf " BYPASS : %d\n" "$BYPS" +echo "" + +# ── 4. Top capabilities ─────────────────────────────────────────────────────── + +echo "=== Top capabilities GET /services ======================================" +extract_param "capability" "${WORK}/svc.log" \ + | sort | uniq -c | sort -rn | head -20 \ + | awk '{printf " %7d %s\n", $1, $2}' +echo "" + +echo "=== Top capabilities GET /devices =======================================" +extract_param "capability" "${WORK}/dev.log" \ + | sort | uniq -c | sort -rn | head -20 \ + | awk '{printf " %7d %s\n", $1, $2}' +echo "" + +# ── 5. Device class and protocol ───────────────────────────────────────────── + +echo "=== Device class breakdown ================================================" +extract_param "deviceClass" "${WORK}/search.log" \ + | sort | uniq -c | sort -rn | head -10 \ + | awk '{printf " %7d %s\n", $1, $2}' +echo "" + +echo "=== Protocol breakdown ====================================================" +extract_param "protocol" "${WORK}/search.log" \ + | sort | uniq -c | sort -rn | head -10 \ + | awk '{printf " %7d %s\n", $1, $2}' +echo "" + +# ── 6. Stage breakdown (services) — canonical form omits PRODUCTION ─────────── + +echo "=== Stage breakdown GET /services (explicit only) =======================" +echo " Note: requests without ?stage= are PRODUCTION (canonical form omits it)" +extract_param "stage" "${WORK}/svc.log" \ + | sort | uniq -c | sort -rn \ + | awk '{printf " %7d %s\n", $1, $2}' +IMPLICIT_PROD=$(extract_qs "${WORK}/svc.log" | grep -vc "stage=" || true) +printf " %7d PRODUCTION (implicit)\n" "$IMPLICIT_PROD" +echo "" + +# ── 7. Top full query string combinations ──────────────────────────────────── + +echo "=== Top 30 query string combinations =====================================" +extract_qs "${WORK}/search.log" \ + | sort | uniq -c | sort -rn | head -30 \ + | awk '{printf " %7d ?%s\n", $1, $2}' +echo "" + +echo "=== Replacements endpoint top parameters =================================" +if [ -s "${WORK}/repl.log" ]; then + extract_qs "${WORK}/repl.log" \ + | sort | uniq -c | sort -rn | head -10 \ + | awk '{printf " %7d ?%s\n", $1, $2}' +else + echo " (no replacement requests in period)" +fi +echo "" + +# ── 8. Push to Prometheus Pushgateway (optional) ───────────────────────────── +# +# Pushes capability counters as Prometheus metrics so they appear as a +# time-series in the dedicated analytics Grafana instance. +# Run weekly via cron to build a trend chart without real-time overhead. + +if [ -n "$PROMETHEUS_PUSH_URL" ]; then + echo "==> Pushing counters to Prometheus Pushgateway ..." + + { + echo "# HELP apix_cdn_capability_requests_total Capability search requests (all, including CDN cache hits)" + echo "# TYPE apix_cdn_capability_requests_total counter" + + extract_param "capability" "${WORK}/svc.log" \ + | sort | uniq -c \ + | awk '{printf "apix_cdn_capability_requests_total{resource=\"services\",capability=\"%s\"} %d\n", $2, $1}' + + extract_param "capability" "${WORK}/dev.log" \ + | sort | uniq -c \ + | awk '{printf "apix_cdn_capability_requests_total{resource=\"devices\",capability=\"%s\"} %d\n", $2, $1}' + + echo "# HELP apix_cdn_cache_requests_total CDN cache disposition for search endpoints" + echo "# TYPE apix_cdn_cache_requests_total counter" + printf "apix_cdn_cache_requests_total{status=\"hit\"} %d\n" "$HITS" + printf "apix_cdn_cache_requests_total{status=\"miss\"} %d\n" "$MISS" + printf "apix_cdn_cache_requests_total{status=\"bypass\"} %d\n" "$BYPS" + + echo "# HELP apix_cdn_deviceclass_requests_total Device class breakdown" + echo "# TYPE apix_cdn_deviceclass_requests_total counter" + extract_param "deviceClass" "${WORK}/search.log" \ + | sort | uniq -c \ + | awk '{printf "apix_cdn_deviceclass_requests_total{deviceClass=\"%s\"} %d\n", $2, $1}' + + echo "# HELP apix_cdn_protocol_requests_total Protocol breakdown" + echo "# TYPE apix_cdn_protocol_requests_total counter" + extract_param "protocol" "${WORK}/search.log" \ + | sort | uniq -c \ + | awk '{printf "apix_cdn_protocol_requests_total{protocol=\"%s\"} %d\n", $2, $1}' + + } | curl -sf --data-binary @- "${PROMETHEUS_PUSH_URL}" && \ + echo " Done." || echo " WARNING: Pushgateway push failed (non-fatal)." + + echo "" +fi + +# ── 9. Summary ──────────────────────────────────────────────────────────────── + +echo "Report complete." +printf "Period : last %d days\n" "$DAYS" +printf "Pull zone : %s\n" "$PULL_ZONE_ID" +[ -n "$PROMETHEUS_PUSH_URL" ] && echo "Pushed to : ${PROMETHEUS_PUSH_URL}" diff --git a/scripts/seed-self-registration.sh b/scripts/seed-self-registration.sh new file mode 100644 index 0000000..989ee27 --- /dev/null +++ b/scripts/seed-self-registration.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Seed RS-01: Bot Standards Foundation (org) + APIX registry (service). +# Run once against a freshly started registry. Safe to re-run — 409s are ignored. +# +# Usage: +# REGISTRY_URL=http://localhost:8180 API_KEY=dev-insecure-key-change-in-prod ./scripts/seed-self-registration.sh + +set -euo pipefail + +REGISTRY_URL="${REGISTRY_URL:-http://localhost:8180}" +API_KEY="${API_KEY:-dev-insecure-key-change-in-prod}" +BSF_EMAIL="${BSF_EMAIL:-carsten@botstandards.org}" + +echo "==> Registering Bot Standards Foundation (org) ..." +ORG_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "${REGISTRY_URL}/organizations" \ + -H "Content-Type: application/json" \ + -H "X-Api-Key: ${API_KEY}" \ + -d '{ + "registrantName": "Bot Standards Foundation", + "registrantEmail": "'"${BSF_EMAIL}"'", + "registrantJurisdiction": "CH", + "registrantOrgType": "FOUNDATION", + "domain": "botstandards.org", + "targetOLevel": "IDENTITY_VERIFIED" + }') + +if [ "${ORG_RESPONSE}" = "409" ]; then + echo " botstandards.org already registered — skipping." +elif [ "${ORG_RESPONSE}" = "201" ] || [ "${ORG_RESPONSE}" = "200" ]; then + echo " Registered. HTTP ${ORG_RESPONSE}" +else + echo " ERROR: HTTP ${ORG_RESPONSE}" >&2 + exit 1 +fi + +echo "" +echo "==> Registering APIX registry service (RS-01) ..." +SVC_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "${REGISTRY_URL}/services" \ + -H "Content-Type: application/json" \ + -H "X-Api-Key: ${API_KEY}" \ + -d '{ + "name": "APIX Registry", + "description": "The open autonomous agent service discovery registry. Agents navigate here to find and invoke any registered service by capability — no hardcoded URLs required.", + "endpoint": "'"${REGISTRY_URL}"'/", + "capabilities": ["service.discovery", "service.query", "service.registration"], + "registrantEmail": "'"${BSF_EMAIL}"'", + "registrantName": "Bot Standards Foundation", + "registrantJurisdiction": "CH", + "registrantOrgType": "FOUNDATION", + "bsmVersion": "0.1", + "serviceStage": "DEVELOPMENT" + }') + +if [ "${SVC_RESPONSE}" = "409" ]; then + echo " APIX registry service already registered — skipping." +elif [ "${SVC_RESPONSE}" = "201" ] || [ "${SVC_RESPONSE}" = "200" ]; then + echo " Registered. HTTP ${SVC_RESPONSE}" +else + echo " ERROR: HTTP ${SVC_RESPONSE}" >&2 + exit 1 +fi + +echo "" +echo "==> Verifying registry root ..." +curl -s "${REGISTRY_URL}/" | python3 -m json.tool 2>/dev/null || \ + curl -s "${REGISTRY_URL}/" + +echo "" +echo "Done. Registry root at ${REGISTRY_URL}/" diff --git a/scripts/setup-bunnynet.sh b/scripts/setup-bunnynet.sh new file mode 100644 index 0000000..b4308a2 --- /dev/null +++ b/scripts/setup-bunnynet.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +# Configure a Bunny.net pull zone for the APIX registry. +# +# What this script does: +# 1. Creates a pull zone pointing at the origin VPS +# 2. Enables query-string vary cache (so /services?capability=nlp and +# /services?capability=translation are cached as separate entries) +# 3. Sets the edge TTL to follow the origin Cache-Control headers +# 4. Adds the custom hostname (api-index.org) +# 5. Prints the CDN hostname to use in your DNS CNAME record +# +# Prerequisites: +# - A Bunny.net account with an API key (bunny.net → Account → API) +# - The origin VPS is reachable at ORIGIN_URL before running this +# - DNS for api-index.org is under your control +# +# Usage: +# BUNNYNET_API_KEY=your-key \ +# ORIGIN_URL=https://your-vps-ip-or-hostname \ +# CUSTOM_HOSTNAME=api-index.org \ +# ./scripts/setup-bunnynet.sh +# +# Optional — enable real-time Loki log forwarding for live telemetry: +# SYSLOG_HOST=your-vps-ip \ +# SYSLOG_PORT=5514 \ +# ./scripts/setup-bunnynet.sh +# +# Optional — install weekly cron job (Mondays 06:00) for CDN analytics report: +# INSTALL_CRON=true \ +# ./scripts/setup-bunnynet.sh +# +# Deploy scripts/promtail-cdn-logs.yaml on the VPS before setting SYSLOG_HOST. +# Bunny.net will start streaming access logs (one syslog line per request, +# including CDN cache hits) to Promtail, which pushes to your Loki instance. +# +# CDN selection rationale: +# Bunny.net is European (Slovenia), GDPR-compliant, has PoPs in Shanghai, +# Singapore, Tokyo, Hong Kong, and Frankfurt. No founding-member conflict +# (Cloudflare and AWS are both founding member candidates and must not +# operate infrastructure over the registry). + +set -euo pipefail + +BUNNYNET_API_KEY="${BUNNYNET_API_KEY:?BUNNYNET_API_KEY is required}" +ORIGIN_URL="${ORIGIN_URL:?ORIGIN_URL is required (e.g. https://1.2.3.4)}" +CUSTOM_HOSTNAME="${CUSTOM_HOSTNAME:-api-index.org}" +PULL_ZONE_NAME="${PULL_ZONE_NAME:-apix-registry}" +BUNNYNET_API="https://api.bunny.net" + +# Optional — set SYSLOG_HOST to enable real-time log forwarding to Promtail/Loki. +# Bunny.net streams one syslog line per request (all requests, not just misses). +# Start Promtail with scripts/promtail-cdn-logs.yaml on the VPS first. +SYSLOG_HOST="${SYSLOG_HOST:-}" +SYSLOG_PORT="${SYSLOG_PORT:-5514}" +INSTALL_CRON="${INSTALL_CRON:-false}" + +echo "==> Creating Bunny.net pull zone '${PULL_ZONE_NAME}' → ${ORIGIN_URL} ..." + +CREATE_RESPONSE=$(curl -sf -X POST "${BUNNYNET_API}/pullzone" \ + -H "AccessKey: ${BUNNYNET_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "Name": "'"${PULL_ZONE_NAME}"'", + "OriginUrl": "'"${ORIGIN_URL}"'", + "Type": 0, + "EnableQueryStringVaryCache": true, + "UseBackgroundUpdate": false, + "EnableWebPVary": false, + "EnableCountryCodeVary": false, + "EnableMobileVary": false, + "EnableHostnameVary": false, + "IgnoreQueryStrings": false, + "CacheControlMaxAgeOverride": -1, + "CacheControlBrowserMaxAgeOverride": -1, + "EnableLogging": true, + "LoggingIPAnonymizationEnabled": true + }') + +# -1 on both MaxAgeOverride fields means "respect the origin Cache-Control header" +# The application sets: GET / → max-age=60, GET /services|/devices → max-age=30 + +PULL_ZONE_ID=$(echo "${CREATE_RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin)['Id'])") +CDN_HOSTNAME=$(echo "${CREATE_RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin)['Hostnames'][0]['Value'])") + +echo " Pull zone ID : ${PULL_ZONE_ID}" +echo " CDN hostname : ${CDN_HOSTNAME}" + +echo "" +echo "==> Adding custom hostname '${CUSTOM_HOSTNAME}' to pull zone ..." + +curl -sf -X POST "${BUNNYNET_API}/pullzone/${PULL_ZONE_ID}/addHostname" \ + -H "AccessKey: ${BUNNYNET_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"Hostname": "'"${CUSTOM_HOSTNAME}"'"}' > /dev/null + +echo " Done." + +echo "" +echo "==> Adding edge rule: bypass cache for internal paths (/q/*) ..." + +curl -sf -X POST "${BUNNYNET_API}/pullzone/${PULL_ZONE_ID}/edgerules/addOrUpdate" \ + -H "AccessKey: ${BUNNYNET_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "Enabled": true, + "Description": "Bypass cache for Quarkus health and metrics endpoints", + "ActionType": 15, + "Triggers": [{ + "Type": 0, + "PatternMatches": ["/q/"], + "PatternMatchingType": 12 + }], + "ActionParameter1": "", + "ActionParameter2": "" + }' > /dev/null + +echo " Done." + +if [ -n "${SYSLOG_HOST}" ]; then + echo "" + echo "==> Enabling real-time log forwarding → ${SYSLOG_HOST}:${SYSLOG_PORT} (TCP Syslog) ..." + + curl -sf -X POST "${BUNNYNET_API}/pullzone/${PULL_ZONE_ID}" \ + -H "AccessKey: ${BUNNYNET_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "LogForwardingEnabled": true, + "LogForwardingHostname": "'"${SYSLOG_HOST}"'", + "LogForwardingPort": '"${SYSLOG_PORT}"', + "LogForwardingProtocol": 1, + "LogForwardingToken": "" + }' > /dev/null + + echo " Done." + echo " Bunny.net will forward every request (hits + misses) as a syslog line." + echo " Promtail → Loki → Grafana will show live capability traffic within seconds." +fi + +echo "" +echo "==> Purging initial cache to ensure clean state ..." + +curl -sf -X POST "${BUNNYNET_API}/pullzone/${PULL_ZONE_ID}/purgeCache" \ + -H "AccessKey: ${BUNNYNET_API_KEY}" > /dev/null + +echo " Done." + +echo "" +echo "======================================================================" +echo " Setup complete. Next steps:" +echo "" +echo " 1. Add a DNS CNAME record:" +echo " ${CUSTOM_HOSTNAME} CNAME ${CDN_HOSTNAME}" +echo "" +echo " 2. Wait for DNS propagation (~5 minutes for most registrars)" +echo "" +echo " 3. Bunny.net will provision a free TLS certificate automatically" +echo " once the CNAME is live. No manual cert management needed." +echo "" +echo " 4. Verify the CDN is caching correctly:" +echo " curl -sI https://${CUSTOM_HOSTNAME}/ | grep -i cache-control" +echo " Expected: Cache-Control: public, max-age=60" +echo "" +echo " 5. Test from Asia (requires a remote machine or VPN):" +echo " curl -w '%{time_total}s\n' -o /dev/null -s https://${CUSTOM_HOSTNAME}/" +echo " First request: ~200ms (cache miss → origin). Subsequent: <20ms (edge)." +echo "" +echo " CDN hostname : ${CDN_HOSTNAME}" +echo " Pull zone ID : ${PULL_ZONE_ID}" +echo "" +echo " Log access (Bunny.net access logs, gzip, one file per day):" +echo " curl -H 'AccessKey: \$BUNNYNET_API_KEY' \\" +echo " https://logging.bunnycdn.com/YYYY/MM/DD/${PULL_ZONE_ID}/ -o logs.gz" +echo "" +echo " Query frequency report (last 7 days):" +echo " BUNNYNET_API_KEY=\$BUNNYNET_API_KEY PULL_ZONE_ID=${PULL_ZONE_ID} \\" +echo " ./scripts/query-report.sh" +echo "" +echo " Logs are IP-anonymised (last octet zeroed) for GDPR compliance." +echo "" +if [ -n "${SYSLOG_HOST}" ]; then +echo " Live Loki forwarding : ENABLED → ${SYSLOG_HOST}:${SYSLOG_PORT}" +echo " Every request (CDN hit and miss) streams as a syslog line." +echo " Grafana dashboard shows capability traffic in real-time." +echo " Promtail config: scripts/promtail-cdn-logs.yaml" +else +echo " Live Loki forwarding : DISABLED (re-run with SYSLOG_HOST= to enable)" +echo " Without this, only cache misses appear in origin Prometheus metrics." +echo " For the OpenClaw demo: enable this before the session starts." +fi +echo "======================================================================" + +# Persist pull zone ID for query-report.sh convenience +echo "${PULL_ZONE_ID}" > .bunnynet-pull-zone-id +echo "" +echo "Pull zone ID saved to .bunnynet-pull-zone-id (add to .gitignore)." + +if [ "${INSTALL_CRON}" = "true" ]; then + echo "" + echo "==> Installing weekly cron job for CDN analytics report ..." + + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPORT_SCRIPT="${SCRIPT_DIR}/query-report.sh" + LOG_DIR="${SCRIPT_DIR}/../logs" + mkdir -p "${LOG_DIR}" + + CRON_LINE="0 6 * * 1 BUNNYNET_API_KEY=${BUNNYNET_API_KEY} PULL_ZONE_ID=${PULL_ZONE_ID} ${REPORT_SCRIPT} >> ${LOG_DIR}/cdn-report.log 2>&1" + + # Install without duplicating — removes any existing query-report.sh entry first + ( crontab -l 2>/dev/null | grep -v "query-report\.sh"; echo "${CRON_LINE}" ) | crontab - + + echo " Done." + echo " Schedule : every Monday at 06:00" + echo " Log : ${LOG_DIR}/cdn-report.log" + echo "" + echo " Verify with: crontab -l | grep query-report" +fi