graylog-deploy/install-graylog.sh

#!/usr/bin/env bash
# Runs INSIDE the Graylog LXC container as root (invoked via `pct exec <vmid> -- bash install-graylog.sh`).
# Idempotent: safe to re-run after a partial failure.
set -euo pipefail

# The container inherits LANG=en_US.UTF-8 from the calling shell (pct exec),
# but that locale is never generated here - just noisy "Setting locale
# failed" warnings from perl/apt-listchanges on every apt-get call. C.UTF-8
# is glibc-builtin (no locale-gen needed) and silences them.
export LC_ALL=C.UTF-8 LANG=C.UTF-8

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
INSTALL_DIR="/opt/graylog"
GRAYLOG_EXTERNAL_URI="${GRAYLOG_EXTERNAL_URI:?Set GRAYLOG_EXTERNAL_URI, e.g. http://93.171.241.5:9000/}"
DISCORD_WEBHOOK_URL="${DISCORD_WEBHOOK_URL:-}"
ADMIN_USER="admin"

# Colors only when stderr is an actual terminal (never garble log files/pipes).
if [ -t 2 ]; then
  C_RESET=$'\033[0m'; C_CYAN=$'\033[36m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m'; C_RED=$'\033[1;31m'
else
  C_RESET=''; C_CYAN=''; C_GREEN=''; C_YELLOW=''; C_RED=''
fi

log()  { echo "${C_CYAN}[install-graylog]${C_RESET} $*" >&2; }
ok()   { echo "${C_GREEN}[install-graylog] ✓${C_RESET} $*" >&2; }
skip() { echo "${C_YELLOW}[install-graylog] ⏭${C_RESET} $*" >&2; }
die()  { echo "${C_RED}[install-graylog] ✗ ERROR:${C_RESET} $*" >&2; exit 1; }

require_root() {
  [ "$(id -u)" -eq 0 ] || die "must run as root inside the container (use pct exec)"
}

# ---------------------------------------------------------------------------
step_packages() {
  log "Installing base packages (curl, gnupg, python3, jq, nftables)..."
  export DEBIAN_FRONTEND=noninteractive
  apt-get update -qq
  apt-get install -y -qq ca-certificates curl gnupg python3 jq nftables >/dev/null
}

step_disable_ipv6() {
  # Workaround: on this hosting network, IPv6 paths to some registries
  # (docker.io) get intercepted and served a mismatched TLS cert. Disabling
  # IPv6 in the container forces IPv4-only egress and avoids it.
  if [ "$(cat /proc/sys/net/ipv6/conf/all/disable_ipv6 2>/dev/null || echo 0)" = "1" ]; then
    skip "IPv6 already disabled"
    return
  fi
  log "Disabling IPv6 (registry TLS workaround for this network)..."
  cat > /etc/sysctl.d/99-disable-ipv6.conf <<'EOF'
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
EOF
  sysctl -p /etc/sysctl.d/99-disable-ipv6.conf >/dev/null
  ok "IPv6 disabled"
}

step_docker_install() {
  if command -v docker >/dev/null 2>&1; then
    skip "Docker already installed ($(docker --version))"
    return
  fi
  log "Installing Docker Engine from the official apt repo..."
  install -m 0755 -d /etc/apt/keyrings
  curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
  chmod a+r /etc/apt/keyrings/docker.asc
  . /etc/os-release
  echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $VERSION_CODENAME stable" \
    > /etc/apt/sources.list.d/docker.list
  apt-get update -qq
  apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin >/dev/null
  systemctl enable --now docker >/dev/null
  ok "Docker Engine installed"
}

step_check_max_map_count() {
  local current
  current="$(cat /proc/sys/vm/max_map_count)"
  if [ "$current" -lt 262144 ]; then
    die "vm.max_map_count=$current (<262144), required by OpenSearch. This is a HOST-level kernel \
parameter and cannot be changed from inside an unprivileged LXC. Ask the Proxmox host admin to run: \
  echo 'vm.max_map_count=262144' >> /etc/sysctl.d/99-opensearch.conf && sysctl -p /etc/sysctl.d/99-opensearch.conf \
Then re-run this script."
  fi
  ok "vm.max_map_count=$current (>= 262144)"
}

step_docker_smoke_test() {
  log "Testing Docker can actually run a container (AppArmor check)..."
  if docker run --rm hello-world >/tmp/docker-test.log 2>&1; then
    ok "Docker smoke test passed"
    return
  fi
  if grep -q "ip_unprivileged_port_start" /tmp/docker-test.log; then
    die "Docker containers fail to start due to unprivileged-LXC AppArmor confinement. \
This must be fixed on the PROXMOX HOST (not from inside the container) by the host admin: \
  echo 'lxc.apparmor.profile: unconfined' >> /etc/pve/lxc/<VMID>.conf && pct reboot <VMID> \
Then re-run this script."
  fi
  cat /tmp/docker-test.log >&2
  die "docker run hello-world failed for an unrecognized reason; see output above."
}

# ---------------------------------------------------------------------------
step_compose_files() {
  mkdir -p "$INSTALL_DIR"
  cp "$SCRIPT_DIR/docker-compose.yml" "$INSTALL_DIR/docker-compose.yml"

  # Same timezone detection used by step_inputs() for Syslog input parsing.
  # GRAYLOG_ROOT_TIMEZONE governs the display timezone of the built-in
  # read-only "admin" user - it CANNOT be changed via the Users REST API
  # (confirmed live: PUT /api/users/admin with a timezone field fails with
  # "state should be: hexString has 24 characters"), only via this env var.
  local tz
  tz="$(cat /etc/timezone 2>/dev/null || echo UTC)"

  if [ -f "$INSTALL_DIR/.env" ]; then
    if grep -q '^GRAYLOG_ROOT_TIMEZONE=' "$INSTALL_DIR/.env"; then
      skip ".env already exists, keeping existing secrets"
    else
      echo "GRAYLOG_ROOT_TIMEZONE=$tz" >> "$INSTALL_DIR/.env"
      ok ".env existed but was missing GRAYLOG_ROOT_TIMEZONE - added it ($tz)"
    fi
    return
  fi
  log "Generating fresh secrets into $INSTALL_DIR/.env ..."
  local secret admin_pass admin_sha2
  secret="$(openssl rand -hex 48)"
  admin_pass="$(openssl rand -base64 18 | tr -dc 'A-Za-z0-9' | head -c20)"
  admin_sha2="$(echo -n "$admin_pass" | sha256sum | cut -d' ' -f1)"

  cat > "$INSTALL_DIR/.env" <<EOF
GRAYLOG_PASSWORD_SECRET=$secret
GRAYLOG_ROOT_PASSWORD_SHA2=$admin_sha2
GRAYLOG_HTTP_EXTERNAL_URI=$GRAYLOG_EXTERNAL_URI
GRAYLOG_ROOT_TIMEZONE=$tz
EOF
  chmod 600 "$INSTALL_DIR/.env"

  # Only shown once, at generation time - not re-printed on later runs.
  cat > "$INSTALL_DIR/.admin_credentials_ONE_TIME" <<EOF
Graylog admin user: $ADMIN_USER
Graylog admin password: $admin_pass
(This file is only written once, at first install. install-graylog.sh prints
this password and deletes this file automatically once the run finishes
successfully. If the script dies before that, delete it yourself once
you've stored the password: rm $INSTALL_DIR/.admin_credentials_ONE_TIME)
EOF
  chmod 600 "$INSTALL_DIR/.admin_credentials_ONE_TIME"
  ok "Admin password generated (shown and deleted at the end of this run if the whole script succeeds)."
}

step_compose_up() {
  log "Pulling images (retrying up to 5x - this registry intermittently serves a bad TLS cert)..."
  local i
  for i in 1 2 3 4 5; do
    if (cd "$INSTALL_DIR" && docker compose pull); then
      break
    fi
    log "Pull attempt $i failed, retrying..."
    sleep 3
    [ "$i" -eq 5 ] && die "docker compose pull failed after 5 attempts."
  done

  log "Starting the stack..."
  (cd "$INSTALL_DIR" && docker compose up -d)

  log "Waiting for graylog-server to become healthy (up to 5 minutes)..."
  local waited=0
  while true; do
    status="$(docker inspect --format '{{.State.Health.Status}}' graylog-server 2>/dev/null || echo starting)"
    [ "$status" = "healthy" ] && break
    waited=$((waited + 5))
    [ "$waited" -ge 300 ] && die "graylog-server did not become healthy within 5 minutes. Check: docker logs graylog-server"
    sleep 5
  done
  ok "Stack is healthy"
}

# ---------------------------------------------------------------------------
GRAYLOG_API="http://localhost:9000/api"

# Resolves the admin password into the global ADMIN_PASSWORD exactly once.
# Must be called as a plain statement (never via `x=$(resolve_admin_password)`)
# so that `die`'s `exit` terminates the real script instead of a subshell.
ADMIN_PASSWORD=""
resolve_admin_password() {
  [ -n "$ADMIN_PASSWORD" ] && return
  if [ -f "$INSTALL_DIR/.admin_credentials_ONE_TIME" ]; then
    ADMIN_PASSWORD="$(grep -oP '(?<=password: ).*' "$INSTALL_DIR/.admin_credentials_ONE_TIME")"
  elif [ -n "${GRAYLOG_ADMIN_PASSWORD:-}" ]; then
    ADMIN_PASSWORD="$GRAYLOG_ADMIN_PASSWORD"
  else
    die "Cannot find admin password: $INSTALL_DIR/.admin_credentials_ONE_TIME was deleted. \
Re-run with GRAYLOG_ADMIN_PASSWORD=<password> set."
  fi
}

gcurl() {
  local method="$1" path="$2" data="${3:-}"
  resolve_admin_password
  if [ -n "$data" ]; then
    curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -H 'X-Requested-By: cli' -H 'Content-Type: application/json' \
      -X "$method" "${GRAYLOG_API}${path}" -d "$data"
  else
    curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -H 'X-Requested-By: cli' -X "$method" "${GRAYLOG_API}${path}"
  fi
}

wait_for_api_ready() {
  # Docker's healthcheck can report the container "healthy" a few seconds
  # before Graylog's REST API is actually ready to serve authenticated
  # requests (indices/auth subsystems still initializing) - confirmed live:
  # the very first gcurl call (step_index_retention) intermittently got an
  # empty response, which then crashed the downstream `python3 -c
  # "json.load(sys.stdin)"` with "Expecting value: line 1 column 1".
  log "Waiting for the Graylog REST API to accept authenticated requests..."
  local waited=0
  while true; do
    gcurl GET /system/indices/index_sets | python3 -c "import json,sys;json.load(sys.stdin)" 2>/dev/null && break
    waited=$((waited + 5))
    [ "$waited" -ge 180 ] && die "Graylog REST API did not respond with valid JSON within 180s of the container reporting healthy."
    sleep 5
  done
  ok "Graylog REST API is ready"
}

step_index_retention() {
  # Graylog's factory default (as of 7.1) keeps 30-40 days of data across up
  # to 20 indices - reasonable in general, but risky on a small disk (this
  # deployment: 50GB) combined with unverified log volume (some sources,
  # e.g. accel-ppp at debug level, can be very chatty). Tightened to a
  # 14-21 day window / 15 indices for a larger safety margin. Revisit once
  # real production volume has been observed for a few weeks.
  log "Tightening default index set retention (idempotent)..."
  local idx_id current
  idx_id="$(gcurl GET /system/indices/index_sets | python3 -c "import json,sys;d=json.load(sys.stdin);print(next(s['id'] for s in d['index_sets'] if s.get('default')))")"
  current="$(gcurl GET "/system/indices/index_sets/$idx_id")"
  if echo "$current" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d['rotation_strategy']['index_lifetime_max']=='P21D' else 1)" 2>/dev/null; then
    skip "index retention already tightened (P14D/P21D, max 15 indices)"
    return
  fi
  gcurl PUT "/system/indices/index_sets/$idx_id" "$(python3 -c "
import json
d = json.loads('''$current''')
d['rotation_strategy']['index_lifetime_min'] = 'P14D'
d['rotation_strategy']['index_lifetime_max'] = 'P21D'
d['retention_strategy']['max_number_of_indices'] = 15
d['data_tiering']['index_lifetime_min'] = 'P14D'
d['data_tiering']['index_lifetime_max'] = 'P21D'
d['writable'] = True
print(json.dumps(d))
")" >/dev/null
  ok "index retention tightened to P14D/P21D, max 15 indices"
}

step_inputs() {
  log "Creating Syslog UDP inputs (idempotent)..."
  local existing tz
  existing="$(gcurl GET /system/inputs)"

  # RFC3164 syslog (which is what most network gear/accel-ppp send) has no
  # timezone in its timestamp ("Jul 22 09:15:13"). Without this setting,
  # Graylog defaults to treating that bare timestamp as UTC - so a device
  # logging in local Kyiv time (UTC+3) shows up 3 hours in the future.
  # Confirmed live: a raw test packet with "Jul 22 09:15:13" was stored as
  # 09:15:13Z UTC (wrong) until this was set; afterwards it correctly
  # stored as 06:15:13Z UTC (09:15:13 Kyiv time). Uses the container's own
  # configured timezone so it stays correct regardless of where this is
  # deployed.
  tz="$(cat /etc/timezone 2>/dev/null || echo UTC)"

  ensure_syslog_input() {
    local port="$1" title="$2" var_name="$3"
    local id current_tz
    id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((i['id'] for i in d['inputs'] if i['attributes'].get('port')==$port),''))")"
    local body="{
      \"title\": \"$title\",
      \"type\": \"org.graylog2.inputs.syslog.udp.SyslogUDPInput\",
      \"global\": true,
      \"configuration\": {\"bind_address\":\"0.0.0.0\",\"port\":$port,\"recv_buffer_size\":262144,
        \"number_worker_threads\":2,\"force_rdns\":false,\"allow_override_date\":true,
        \"store_full_message\":false,\"expand_structured_data\":true,\"charset_name\":\"UTF-8\",
        \"timezone\":\"$tz\"}
    }"
    if [ -z "$id" ]; then
      id="$(gcurl POST /system/inputs "$body" | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")"
      ok "Created $title: $id"
    else
      current_tz="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((i['attributes'].get('timezone') for i in d['inputs'] if i['attributes'].get('port')==$port),''))")"
      if [ "$current_tz" != "$tz" ]; then
        gcurl PUT "/system/inputs/$id" "$body" >/dev/null
        ok "$title already existed - fixed timezone ($current_tz -> $tz)"
      else
        skip "$title already exists: $id"
      fi
    fi
    printf -v "$var_name" '%s' "$id"
  }

  # Port 514 is the standard syslog port and the one most network gear
  # actually sends to (confirmed live: BDCOM switches here can't be pointed
  # at a custom port). Port 1514 is kept as a secondary input for any
  # equipment that *can* be configured with a non-standard destination port.
  ensure_syslog_input 514 "Network Equipment Syslog (standard port 514)" NETWORK_INPUT_ID_514
  ensure_syslog_input 1514 "Network Equipment Syslog (Juniper-ZTE-DLink)" NETWORK_INPUT_ID_1514
  ensure_syslog_input 5140 "Servers Syslog (RADIUS-accel-ppp)" SERVERS_INPUT_ID
}

step_pipeline_rules() {
  log "Importing pipeline rules from $SCRIPT_DIR/rules/*.json (idempotent)..."
  local existing_titles
  existing_titles="$(gcurl GET /system/pipelines/rule | python3 -c "import json,sys;print('\n'.join(r['title'] for r in json.load(sys.stdin)))")"

  local f title
  for f in "$SCRIPT_DIR"/rules/*.json; do
    title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")"
    if echo "$existing_titles" | grep -qx "$title"; then
      skip "rule '$title' already exists"
      continue
    fi
    local result
    result="$(gcurl POST /system/pipelines/rule "$(cat "$f")")"
    if echo "$result" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d.get('errors') is None else 1)"; then
      ok "created rule '$title'"
    else
      die "rule '$title' failed to compile: $result"
    fi
  done
}

step_pipelines() {
  log "Importing pipelines from $SCRIPT_DIR/pipelines/*.json (idempotent)..."
  local existing
  existing="$(gcurl GET /system/pipelines/pipeline)"

  local f title id
  for f in "$SCRIPT_DIR"/pipelines/*.json; do
    title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")"
    id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((p['id'] for p in d if p['title']=='$title'),''))")"
    if [ -n "$id" ]; then
      skip "pipeline '$title' already exists ($id)"
      if [ "$title" = "Network Equipment Parsing" ]; then NETWORK_PIPELINE_ID="$id"; fi
      if [ "$title" = "Servers Parsing" ]; then SERVERS_PIPELINE_ID="$id"; fi
      continue
    fi
    result="$(gcurl POST /system/pipelines/pipeline "$(cat "$f")")"
    id="$(echo "$result" | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")"
    [ -n "$id" ] || die "pipeline '$title' failed to create: $result"
    ok "created pipeline '$title' ($id)"
    if [ "$title" = "Network Equipment Parsing" ]; then NETWORK_PIPELINE_ID="$id"; fi
    if [ "$title" = "Servers Parsing" ]; then SERVERS_PIPELINE_ID="$id"; fi
  done
}

step_streams() {
  log "Creating streams from $SCRIPT_DIR/streams/*.json (idempotent)..."
  local index_set_id existing
  index_set_id="$(gcurl GET /system/indices/index_sets | python3 -c "import json,sys;d=json.load(sys.stdin);print(next(s['id'] for s in d['index_sets'] if s.get('default')))")"
  existing="$(gcurl GET /streams)"

  # $2.. are extra `sed -e` substitution expressions applied on top of the
  # index-set-id one, so each template can carry however many input-id
  # placeholders it needs.
  render_and_create() {
    local tmpl="$1" title="$2"; shift 2
    local id
    id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((s['id'] for s in d['streams'] if s['title']=='$title'),''))")"
    if [ -n "$id" ]; then
      skip "stream '$title' already exists ($id)"
      echo "$id"
      return
    fi
    local sed_args=(-e "s/__DEFAULT_INDEX_SET_ID__/$index_set_id/")
    local expr
    for expr in "$@"; do sed_args+=(-e "$expr"); done
    local body
    body="$(sed "${sed_args[@]}" "$tmpl")"
    id="$(gcurl POST /streams "$body" | python3 -c "import json,sys;print(json.load(sys.stdin)['stream_id'])")"
    ok "created stream '$title' ($id)"
    gcurl POST "/streams/$id/resume" "" >/dev/null
    echo "$id"
  }

  NETWORK_STREAM_ID="$(render_and_create "$SCRIPT_DIR/streams/stream1_network.json" "Network Equipment" \
    "s/__NETWORK_INPUT_ID_514__/$NETWORK_INPUT_ID_514/" \
    "s/__NETWORK_INPUT_ID_1514__/$NETWORK_INPUT_ID_1514/")"
  SERVERS_STREAM_ID="$(render_and_create "$SCRIPT_DIR/streams/stream2_servers.json" "Servers" \
    "s/__SERVERS_INPUT_ID__/$SERVERS_INPUT_ID/")"
}

step_connect_pipelines() {
  log "Connecting pipelines to streams..."
  gcurl POST /system/pipelines/connections/to_stream \
    "{\"stream_id\":\"$NETWORK_STREAM_ID\",\"pipeline_ids\":[\"$NETWORK_PIPELINE_ID\"]}" >/dev/null
  gcurl POST /system/pipelines/connections/to_stream \
    "{\"stream_id\":\"$SERVERS_STREAM_ID\",\"pipeline_ids\":[\"$SERVERS_PIPELINE_ID\"]}" >/dev/null
  ok "Pipelines connected to streams"
}

step_firewall() {
  cp "$SCRIPT_DIR/nftables.conf" /etc/nftables.conf
  systemctl enable nftables >/dev/null 2>&1 || true
  nft -f /etc/nftables.conf
  ok "nftables applied (only 9000/tcp, 514+1514/udp, 5140/udp open)"
}

step_dashboard() {
  # Three focused dashboards instead of one combined view - each targets a
  # different reader (landing/alerts overview, network-equipment-only,
  # servers-only) so opening Graylog goes straight to what's relevant
  # instead of one big mixed-stream page.
  local existing_titles
  existing_titles="$(gcurl GET /views | python3 -c "import json,sys;print('\n'.join(v['title'] for v in json.load(sys.stdin)['views']))")"

  local search_file view_file title id search_id body
  for search_file in "$SCRIPT_DIR"/dashboards/search_*.json; do
    view_file="${search_file/search_/view_}"
    [ -f "$view_file" ] || die "missing $view_file for $search_file"
    title="$(python3 -c "import json;print(json.load(open('$view_file'))['entity']['title'])")"
    if echo "$existing_titles" | grep -qx "$title"; then
      skip "dashboard '$title' already exists"
      continue
    fi

    body="$(sed -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/g" -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/g" "$search_file")"
    search_id="$(gcurl POST /views/search "$body" | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")"
    [ -n "$search_id" ] || die "search creation failed for dashboard '$title'"

    body="$(sed -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/g" -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/g" -e "s/__SEARCH_ID__/$search_id/" "$view_file")"
    id="$(gcurl POST /views "$body" | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")"
    [ -n "$id" ] || die "dashboard '$title' failed to create"
    ok "created dashboard '$title' ($id)"
  done
}

step_alerts() {
  if [ -z "$DISCORD_WEBHOOK_URL" ]; then
    skip "Discord notification and alerts (DISCORD_WEBHOOK_URL not set)"
    return
  fi
  if [ ! -f "$SCRIPT_DIR/alerts/notification_discord.json" ]; then
    skip "Alerts (no alerts/notification_discord.json in this checkout)"
    return
  fi

  log "Creating Discord notification (idempotent)..."
  local existing_notifs notif_id
  existing_notifs="$(gcurl GET /events/notifications)"
  notif_id="$(echo "$existing_notifs" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((n['id'] for n in d['notifications'] if n['title']=='Discord - Critical Alerts'),''))")"
  if [ -n "$notif_id" ]; then
    skip "Discord notification already exists ($notif_id)"
  else
    local webhook_slack body
    webhook_slack="${DISCORD_WEBHOOK_URL%/}/slack"
    body="$(sed "s#__DISCORD_WEBHOOK_URL_SLACK__#${webhook_slack}#" "$SCRIPT_DIR/alerts/notification_discord.json")"
    notif_id="$(gcurl POST /events/notifications "{\"entity\": ${body}, \"share_request\": {\"selected_grantee_capabilities\": {}}}" \
      | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")"
    ok "created Discord notification ($notif_id)"
  fi

  log "Creating alert (event) definitions from $SCRIPT_DIR/alerts/*.json (idempotent)..."
  local existing_defs
  existing_defs="$(gcurl GET /events/definitions)"
  local f title id
  for f in "$SCRIPT_DIR"/alerts/alert*.json; do
    [ -f "$f" ] || continue
    title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")"
    id="$(echo "$existing_defs" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((e['id'] for e in d['event_definitions'] if e['title']=='$title'),''))")"
    if [ -n "$id" ]; then
      skip "alert '$title' already exists ($id)"
      continue
    fi
    local body
    body="$(sed \
      -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/" \
      -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/" \
      -e "s/__DISCORD_NOTIFICATION_ID__/$notif_id/" \
      "$f")"
    id="$(gcurl POST /events/definitions "{\"entity\": ${body}, \"share_request\": {\"selected_grantee_capabilities\": {}}}" \
      | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")"
    [ -n "$id" ] || die "alert '$title' failed to create"
    gcurl PUT "/events/definitions/$id/schedule" "" >/dev/null
    ok "created and enabled alert '$title' ($id)"
  done
}

# ---------------------------------------------------------------------------
main() {
  require_root
  step_packages
  step_disable_ipv6
  step_docker_install
  step_check_max_map_count
  step_docker_smoke_test
  step_firewall
  step_compose_files
  step_compose_up

  # Must be a plain statement (not `x=$(...)`) so a failure here truly
  # aborts the script - gcurl() is always called from inside command
  # substitutions downstream, where `die`'s `exit` would only kill a subshell.
  resolve_admin_password
  wait_for_api_ready

  step_index_retention
  step_inputs
  step_pipeline_rules
  step_pipelines
  step_streams
  step_connect_pipelines
  step_dashboard
  step_alerts

  echo >&2
  echo "${C_GREEN}==================================================${C_RESET}" >&2
  echo "${C_GREEN} DONE - Graylog is up${C_RESET}" >&2
  echo "${C_GREEN}==================================================${C_RESET}" >&2
  echo " Web UI: $GRAYLOG_EXTERNAL_URI" >&2
  echo " Config: $INSTALL_DIR/docker-compose.yml and $INSTALL_DIR/.env" >&2
  if [ -f "$INSTALL_DIR/.admin_credentials_ONE_TIME" ]; then
    echo " Admin user:     $ADMIN_USER" >&2
    echo " Admin password: $ADMIN_PASSWORD" >&2
    echo " (shown once above - store it now; the one-time file is being deleted)" >&2
    rm -f "$INSTALL_DIR/.admin_credentials_ONE_TIME"
  fi
  echo "${C_GREEN}==================================================${C_RESET}" >&2
}

main "$@"