Carsten Rehfeld
226 lines
9.7 KiB
Bash
226 lines
9.7 KiB
Bash
#!/usr/bin/env bash
|
|
# Download Bunny.net access logs and produce a capability frequency report.
|
|
#
|
|
# This is the only way to see CDN-cached request frequency — the origin
|
|
# Prometheus metrics only see cache misses. A weekly cron run pushing the
|
|
# output to Grafana covers >99% of analysis needs without real-time overhead.
|
|
#
|
|
# What this script answers:
|
|
# - Which capabilities are queried most often (all requests, including hits)?
|
|
# - Which device class / protocol combinations are active?
|
|
# - What is the cache hit ratio per endpoint?
|
|
# - Which full query string combinations are most common?
|
|
#
|
|
# Usage:
|
|
# BUNNYNET_API_KEY=your-key PULL_ZONE_ID=12345 ./scripts/query-report.sh
|
|
# BUNNYNET_API_KEY=your-key PULL_ZONE_ID=12345 DAYS=30 ./scripts/query-report.sh
|
|
#
|
|
# Output:
|
|
# Human-readable report to stdout.
|
|
# With PROMETHEUS_PUSH_URL set, also pushes counters to a Prometheus
|
|
# Pushgateway (e.g. your dedicated analytics Grafana stack) so the weekly
|
|
# numbers become a time-series rather than a point-in-time dump.
|
|
#
|
|
# PROMETHEUS_PUSH_URL=https://pushgateway.your-grafana.example/metrics/job/apix-cdn-report \
|
|
# BUNNYNET_API_KEY=your-key PULL_ZONE_ID=12345 ./scripts/query-report.sh
|
|
#
|
|
# Prerequisites: curl, gzip, awk (any POSIX awk)
|
|
|
|
set -euo pipefail
|
|
|
|
BUNNYNET_API_KEY="${BUNNYNET_API_KEY:?BUNNYNET_API_KEY is required}"
|
|
PULL_ZONE_ID="${PULL_ZONE_ID:-$(cat .bunnynet-pull-zone-id 2>/dev/null || echo '')}"
|
|
PULL_ZONE_ID="${PULL_ZONE_ID:?PULL_ZONE_ID is required (or run setup-bunnynet.sh first)}"
|
|
DAYS="${DAYS:-7}"
|
|
PROMETHEUS_PUSH_URL="${PROMETHEUS_PUSH_URL:-}"
|
|
LOGGING_BASE="https://logging.bunnycdn.com"
|
|
|
|
WORK=$(mktemp -d)
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
touch "${WORK}/combined.log"
|
|
|
|
# ── 1. Download logs ──────────────────────────────────────────────────────────
|
|
|
|
echo "==> Downloading Bunny.net access logs — pull zone ${PULL_ZONE_ID}, last ${DAYS} days ..."
|
|
|
|
for i in $(seq 1 "$DAYS"); do
|
|
# Support both GNU date (Linux) and BSD date (macOS)
|
|
if date --version >/dev/null 2>&1; then
|
|
DATE=$(date -d "-${i} days" +%Y/%m/%d)
|
|
else
|
|
DATE=$(date -v "-${i}d" +%Y/%m/%d)
|
|
fi
|
|
Y=$(echo "$DATE" | cut -d/ -f1)
|
|
M=$(echo "$DATE" | cut -d/ -f2)
|
|
D=$(echo "$DATE" | cut -d/ -f3)
|
|
|
|
GZ="${WORK}/${Y}-${M}-${D}.gz"
|
|
STATUS=$(curl -s -o "$GZ" -w "%{http_code}" \
|
|
-H "AccessKey: ${BUNNYNET_API_KEY}" \
|
|
"${LOGGING_BASE}/${Y}/${M}/${D}/${PULL_ZONE_ID}/")
|
|
|
|
if [ "$STATUS" = "200" ] && [ -s "$GZ" ]; then
|
|
gunzip -c "$GZ" >> "${WORK}/combined.log"
|
|
printf " %s-%s-%s OK\n" "$Y" "$M" "$D"
|
|
else
|
|
printf " %s-%s-%s no data (HTTP %s)\n" "$Y" "$M" "$D" "$STATUS"
|
|
fi
|
|
done
|
|
|
|
if [ ! -s "${WORK}/combined.log" ]; then
|
|
echo ""
|
|
echo "No log data found for the requested period."
|
|
exit 0
|
|
fi
|
|
|
|
# ── 2. Isolate search requests ────────────────────────────────────────────────
|
|
|
|
grep -E '"GET /services\?' "${WORK}/combined.log" > "${WORK}/svc.log" || true
|
|
grep -E '"GET /devices\?' "${WORK}/combined.log" > "${WORK}/dev.log" || true
|
|
grep -E '"GET /devices\?|"GET /services\?' "${WORK}/combined.log" > "${WORK}/search.log" || true
|
|
grep -E '"GET /[^"?]+/replacements\?' "${WORK}/combined.log" > "${WORK}/repl.log" || true
|
|
|
|
TOTAL=$(wc -l < "${WORK}/combined.log")
|
|
SEARCH=$(wc -l < "${WORK}/search.log")
|
|
|
|
echo ""
|
|
printf "Total log entries : %d\n" "$TOTAL"
|
|
printf "Search requests : %d (GET /services + GET /devices)\n" "$SEARCH"
|
|
echo ""
|
|
|
|
# ── Helper: extract query string from a log line ──────────────────────────────
|
|
# Input line: 1.2.3.x - - [date] "GET /services?capability=nlp HTTP/1.1" 200 … HIT
|
|
# Output: capability=nlp
|
|
|
|
extract_qs() {
|
|
awk '{
|
|
if (match($0, /"GET [^?]+"?\?([^" ]+) HTTP/, arr)) print arr[1]
|
|
}' "$1"
|
|
}
|
|
|
|
extract_param() {
|
|
local param="$1" file="$2"
|
|
extract_qs "$file" | tr "&" "\n" | grep "^${param}=" | cut -d= -f2-
|
|
}
|
|
|
|
# ── 3. Cache hit ratio ────────────────────────────────────────────────────────
|
|
|
|
echo "=== Cache hit ratio (search endpoints) ==================================="
|
|
HITS=$(awk '$NF=="HIT"' "${WORK}/search.log" | wc -l || echo 0)
|
|
MISS=$(awk '$NF=="MISS"' "${WORK}/search.log" | wc -l || echo 0)
|
|
BYPS=$(awk '$NF=="BYPASS"' "${WORK}/search.log" | wc -l || echo 0)
|
|
printf " HIT : %d\n" "$HITS"
|
|
printf " MISS : %d\n" "$MISS"
|
|
printf " BYPASS : %d\n" "$BYPS"
|
|
echo ""
|
|
|
|
# ── 4. Top capabilities ───────────────────────────────────────────────────────
|
|
|
|
echo "=== Top capabilities GET /services ======================================"
|
|
extract_param "capability" "${WORK}/svc.log" \
|
|
| sort | uniq -c | sort -rn | head -20 \
|
|
| awk '{printf " %7d %s\n", $1, $2}'
|
|
echo ""
|
|
|
|
echo "=== Top capabilities GET /devices ======================================="
|
|
extract_param "capability" "${WORK}/dev.log" \
|
|
| sort | uniq -c | sort -rn | head -20 \
|
|
| awk '{printf " %7d %s\n", $1, $2}'
|
|
echo ""
|
|
|
|
# ── 5. Device class and protocol ─────────────────────────────────────────────
|
|
|
|
echo "=== Device class breakdown ================================================"
|
|
extract_param "deviceClass" "${WORK}/search.log" \
|
|
| sort | uniq -c | sort -rn | head -10 \
|
|
| awk '{printf " %7d %s\n", $1, $2}'
|
|
echo ""
|
|
|
|
echo "=== Protocol breakdown ===================================================="
|
|
extract_param "protocol" "${WORK}/search.log" \
|
|
| sort | uniq -c | sort -rn | head -10 \
|
|
| awk '{printf " %7d %s\n", $1, $2}'
|
|
echo ""
|
|
|
|
# ── 6. Stage breakdown (services) — canonical form omits PRODUCTION ───────────
|
|
|
|
echo "=== Stage breakdown GET /services (explicit only) ======================="
|
|
echo " Note: requests without ?stage= are PRODUCTION (canonical form omits it)"
|
|
extract_param "stage" "${WORK}/svc.log" \
|
|
| sort | uniq -c | sort -rn \
|
|
| awk '{printf " %7d %s\n", $1, $2}'
|
|
IMPLICIT_PROD=$(extract_qs "${WORK}/svc.log" | grep -vc "stage=" || true)
|
|
printf " %7d PRODUCTION (implicit)\n" "$IMPLICIT_PROD"
|
|
echo ""
|
|
|
|
# ── 7. Top full query string combinations ────────────────────────────────────
|
|
|
|
echo "=== Top 30 query string combinations ====================================="
|
|
extract_qs "${WORK}/search.log" \
|
|
| sort | uniq -c | sort -rn | head -30 \
|
|
| awk '{printf " %7d ?%s\n", $1, $2}'
|
|
echo ""
|
|
|
|
echo "=== Replacements endpoint top parameters ================================="
|
|
if [ -s "${WORK}/repl.log" ]; then
|
|
extract_qs "${WORK}/repl.log" \
|
|
| sort | uniq -c | sort -rn | head -10 \
|
|
| awk '{printf " %7d ?%s\n", $1, $2}'
|
|
else
|
|
echo " (no replacement requests in period)"
|
|
fi
|
|
echo ""
|
|
|
|
# ── 8. Push to Prometheus Pushgateway (optional) ─────────────────────────────
|
|
#
|
|
# Pushes capability counters as Prometheus metrics so they appear as a
|
|
# time-series in the dedicated analytics Grafana instance.
|
|
# Run weekly via cron to build a trend chart without real-time overhead.
|
|
|
|
if [ -n "$PROMETHEUS_PUSH_URL" ]; then
|
|
echo "==> Pushing counters to Prometheus Pushgateway ..."
|
|
|
|
{
|
|
echo "# HELP apix_cdn_capability_requests_total Capability search requests (all, including CDN cache hits)"
|
|
echo "# TYPE apix_cdn_capability_requests_total counter"
|
|
|
|
extract_param "capability" "${WORK}/svc.log" \
|
|
| sort | uniq -c \
|
|
| awk '{printf "apix_cdn_capability_requests_total{resource=\"services\",capability=\"%s\"} %d\n", $2, $1}'
|
|
|
|
extract_param "capability" "${WORK}/dev.log" \
|
|
| sort | uniq -c \
|
|
| awk '{printf "apix_cdn_capability_requests_total{resource=\"devices\",capability=\"%s\"} %d\n", $2, $1}'
|
|
|
|
echo "# HELP apix_cdn_cache_requests_total CDN cache disposition for search endpoints"
|
|
echo "# TYPE apix_cdn_cache_requests_total counter"
|
|
printf "apix_cdn_cache_requests_total{status=\"hit\"} %d\n" "$HITS"
|
|
printf "apix_cdn_cache_requests_total{status=\"miss\"} %d\n" "$MISS"
|
|
printf "apix_cdn_cache_requests_total{status=\"bypass\"} %d\n" "$BYPS"
|
|
|
|
echo "# HELP apix_cdn_deviceclass_requests_total Device class breakdown"
|
|
echo "# TYPE apix_cdn_deviceclass_requests_total counter"
|
|
extract_param "deviceClass" "${WORK}/search.log" \
|
|
| sort | uniq -c \
|
|
| awk '{printf "apix_cdn_deviceclass_requests_total{deviceClass=\"%s\"} %d\n", $2, $1}'
|
|
|
|
echo "# HELP apix_cdn_protocol_requests_total Protocol breakdown"
|
|
echo "# TYPE apix_cdn_protocol_requests_total counter"
|
|
extract_param "protocol" "${WORK}/search.log" \
|
|
| sort | uniq -c \
|
|
| awk '{printf "apix_cdn_protocol_requests_total{protocol=\"%s\"} %d\n", $2, $1}'
|
|
|
|
} | curl -sf --data-binary @- "${PROMETHEUS_PUSH_URL}" && \
|
|
echo " Done." || echo " WARNING: Pushgateway push failed (non-fatal)."
|
|
|
|
echo ""
|
|
fi
|
|
|
|
# ── 9. Summary ────────────────────────────────────────────────────────────────
|
|
|
|
echo "Report complete."
|
|
printf "Period : last %d days\n" "$DAYS"
|
|
printf "Pull zone : %s\n" "$PULL_ZONE_ID"
|
|
[ -n "$PROMETHEUS_PUSH_URL" ] && echo "Pushed to : ${PROMETHEUS_PUSH_URL}"
|