#!/usr/bin/env bash # Runs INSIDE the Graylog LXC container as root (invoked via `pct exec -- bash install-graylog.sh`). # Idempotent: safe to re-run after a partial failure. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INSTALL_DIR="/opt/graylog" GRAYLOG_EXTERNAL_URI="${GRAYLOG_EXTERNAL_URI:?Set GRAYLOG_EXTERNAL_URI, e.g. http://93.171.241.5:9000/}" DISCORD_WEBHOOK_URL="${DISCORD_WEBHOOK_URL:-}" ADMIN_USER="admin" # Colors only when stderr is an actual terminal (never garble log files/pipes). if [ -t 2 ]; then C_RESET=$'\033[0m'; C_CYAN=$'\033[36m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m'; C_RED=$'\033[1;31m' else C_RESET=''; C_CYAN=''; C_GREEN=''; C_YELLOW=''; C_RED='' fi log() { echo "${C_CYAN}[install-graylog]${C_RESET} $*" >&2; } ok() { echo "${C_GREEN}[install-graylog] ✓${C_RESET} $*" >&2; } skip() { echo "${C_YELLOW}[install-graylog] ⏭${C_RESET} $*" >&2; } die() { echo "${C_RED}[install-graylog] ✗ ERROR:${C_RESET} $*" >&2; exit 1; } require_root() { [ "$(id -u)" -eq 0 ] || die "must run as root inside the container (use pct exec)" } # --------------------------------------------------------------------------- step_packages() { log "Installing base packages (curl, gnupg, python3, jq, nftables)..." export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get install -y -qq ca-certificates curl gnupg python3 jq nftables >/dev/null } step_disable_ipv6() { # Workaround: on this hosting network, IPv6 paths to some registries # (docker.io) get intercepted and served a mismatched TLS cert. Disabling # IPv6 in the container forces IPv4-only egress and avoids it. if [ "$(cat /proc/sys/net/ipv6/conf/all/disable_ipv6 2>/dev/null || echo 0)" = "1" ]; then skip "IPv6 already disabled" return fi log "Disabling IPv6 (registry TLS workaround for this network)..." cat > /etc/sysctl.d/99-disable-ipv6.conf <<'EOF' net.ipv6.conf.all.disable_ipv6 = 1 net.ipv6.conf.default.disable_ipv6 = 1 EOF sysctl -p /etc/sysctl.d/99-disable-ipv6.conf >/dev/null ok "IPv6 disabled" } step_docker_install() { if command -v docker >/dev/null 2>&1; then skip "Docker already installed ($(docker --version))" return fi log "Installing Docker Engine from the official apt repo..." install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc . /etc/os-release echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $VERSION_CODENAME stable" \ > /etc/apt/sources.list.d/docker.list apt-get update -qq apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin >/dev/null systemctl enable --now docker >/dev/null ok "Docker Engine installed" } step_check_max_map_count() { local current current="$(cat /proc/sys/vm/max_map_count)" if [ "$current" -lt 262144 ]; then die "vm.max_map_count=$current (<262144), required by OpenSearch. This is a HOST-level kernel \ parameter and cannot be changed from inside an unprivileged LXC. Ask the Proxmox host admin to run: \ echo 'vm.max_map_count=262144' >> /etc/sysctl.d/99-opensearch.conf && sysctl -p /etc/sysctl.d/99-opensearch.conf \ Then re-run this script." fi ok "vm.max_map_count=$current (>= 262144)" } step_docker_smoke_test() { log "Testing Docker can actually run a container (AppArmor check)..." if docker run --rm hello-world >/tmp/docker-test.log 2>&1; then ok "Docker smoke test passed" return fi if grep -q "ip_unprivileged_port_start" /tmp/docker-test.log; then die "Docker containers fail to start due to unprivileged-LXC AppArmor confinement. \ This must be fixed on the PROXMOX HOST (not from inside the container) by the host admin: \ echo 'lxc.apparmor.profile: unconfined' >> /etc/pve/lxc/.conf && pct reboot \ Then re-run this script." fi cat /tmp/docker-test.log >&2 die "docker run hello-world failed for an unrecognized reason; see output above." } # --------------------------------------------------------------------------- step_compose_files() { mkdir -p "$INSTALL_DIR" cp "$SCRIPT_DIR/docker-compose.yml" "$INSTALL_DIR/docker-compose.yml" # Same timezone detection used by step_inputs() for Syslog input parsing. # GRAYLOG_ROOT_TIMEZONE governs the display timezone of the built-in # read-only "admin" user - it CANNOT be changed via the Users REST API # (confirmed live: PUT /api/users/admin with a timezone field fails with # "state should be: hexString has 24 characters"), only via this env var. local tz tz="$(cat /etc/timezone 2>/dev/null || echo UTC)" if [ -f "$INSTALL_DIR/.env" ]; then if grep -q '^GRAYLOG_ROOT_TIMEZONE=' "$INSTALL_DIR/.env"; then skip ".env already exists, keeping existing secrets" else echo "GRAYLOG_ROOT_TIMEZONE=$tz" >> "$INSTALL_DIR/.env" ok ".env existed but was missing GRAYLOG_ROOT_TIMEZONE - added it ($tz)" fi return fi log "Generating fresh secrets into $INSTALL_DIR/.env ..." local secret admin_pass admin_sha2 secret="$(openssl rand -hex 48)" admin_pass="$(openssl rand -base64 18 | tr -dc 'A-Za-z0-9' | head -c20)" admin_sha2="$(echo -n "$admin_pass" | sha256sum | cut -d' ' -f1)" cat > "$INSTALL_DIR/.env" < "$INSTALL_DIR/.admin_credentials_ONE_TIME" < set." fi } gcurl() { local method="$1" path="$2" data="${3:-}" resolve_admin_password if [ -n "$data" ]; then curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -H 'X-Requested-By: cli' -H 'Content-Type: application/json' \ -X "$method" "${GRAYLOG_API}${path}" -d "$data" else curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -H 'X-Requested-By: cli' -X "$method" "${GRAYLOG_API}${path}" fi } step_index_retention() { # Graylog's factory default (as of 7.1) keeps 30-40 days of data across up # to 20 indices - reasonable in general, but risky on a small disk (this # deployment: 50GB) combined with unverified log volume (some sources, # e.g. accel-ppp at debug level, can be very chatty). Tightened to a # 14-21 day window / 15 indices for a larger safety margin. Revisit once # real production volume has been observed for a few weeks. log "Tightening default index set retention (idempotent)..." local idx_id current idx_id="$(gcurl GET /system/indices/index_sets | python3 -c "import json,sys;d=json.load(sys.stdin);print(next(s['id'] for s in d['index_sets'] if s.get('default')))")" current="$(gcurl GET "/system/indices/index_sets/$idx_id")" if echo "$current" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d['rotation_strategy']['index_lifetime_max']=='P21D' else 1)" 2>/dev/null; then skip "index retention already tightened (P14D/P21D, max 15 indices)" return fi gcurl PUT "/system/indices/index_sets/$idx_id" "$(python3 -c " import json d = json.loads('''$current''') d['rotation_strategy']['index_lifetime_min'] = 'P14D' d['rotation_strategy']['index_lifetime_max'] = 'P21D' d['retention_strategy']['max_number_of_indices'] = 15 d['data_tiering']['index_lifetime_min'] = 'P14D' d['data_tiering']['index_lifetime_max'] = 'P21D' d['writable'] = True print(json.dumps(d)) ")" >/dev/null ok "index retention tightened to P14D/P21D, max 15 indices" } step_inputs() { log "Creating Syslog UDP inputs (idempotent)..." local existing tz existing="$(gcurl GET /system/inputs)" # RFC3164 syslog (which is what most network gear/accel-ppp send) has no # timezone in its timestamp ("Jul 22 09:15:13"). Without this setting, # Graylog defaults to treating that bare timestamp as UTC - so a device # logging in local Kyiv time (UTC+3) shows up 3 hours in the future. # Confirmed live: a raw test packet with "Jul 22 09:15:13" was stored as # 09:15:13Z UTC (wrong) until this was set; afterwards it correctly # stored as 06:15:13Z UTC (09:15:13 Kyiv time). Uses the container's own # configured timezone so it stays correct regardless of where this is # deployed. tz="$(cat /etc/timezone 2>/dev/null || echo UTC)" ensure_syslog_input() { local port="$1" title="$2" var_name="$3" local id current_tz id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((i['id'] for i in d['inputs'] if i['attributes'].get('port')==$port),''))")" local body="{ \"title\": \"$title\", \"type\": \"org.graylog2.inputs.syslog.udp.SyslogUDPInput\", \"global\": true, \"configuration\": {\"bind_address\":\"0.0.0.0\",\"port\":$port,\"recv_buffer_size\":262144, \"number_worker_threads\":2,\"force_rdns\":false,\"allow_override_date\":true, \"store_full_message\":false,\"expand_structured_data\":true,\"charset_name\":\"UTF-8\", \"timezone\":\"$tz\"} }" if [ -z "$id" ]; then id="$(gcurl POST /system/inputs "$body" | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")" ok "Created $title: $id" else current_tz="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((i['attributes'].get('timezone') for i in d['inputs'] if i['attributes'].get('port')==$port),''))")" if [ "$current_tz" != "$tz" ]; then gcurl PUT "/system/inputs/$id" "$body" >/dev/null ok "$title already existed - fixed timezone ($current_tz -> $tz)" else skip "$title already exists: $id" fi fi printf -v "$var_name" '%s' "$id" } # Port 514 is the standard syslog port and the one most network gear # actually sends to (confirmed live: BDCOM switches here can't be pointed # at a custom port). Port 1514 is kept as a secondary input for any # equipment that *can* be configured with a non-standard destination port. ensure_syslog_input 514 "Network Equipment Syslog (standard port 514)" NETWORK_INPUT_ID_514 ensure_syslog_input 1514 "Network Equipment Syslog (Juniper-ZTE-DLink)" NETWORK_INPUT_ID_1514 ensure_syslog_input 5140 "Servers Syslog (RADIUS-accel-ppp)" SERVERS_INPUT_ID } step_pipeline_rules() { log "Importing pipeline rules from $SCRIPT_DIR/rules/*.json (idempotent)..." local existing_titles existing_titles="$(gcurl GET /system/pipelines/rule | python3 -c "import json,sys;print('\n'.join(r['title'] for r in json.load(sys.stdin)))")" local f title for f in "$SCRIPT_DIR"/rules/*.json; do title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")" if echo "$existing_titles" | grep -qx "$title"; then skip "rule '$title' already exists" continue fi local result result="$(gcurl POST /system/pipelines/rule "$(cat "$f")")" if echo "$result" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d.get('errors') is None else 1)"; then ok "created rule '$title'" else die "rule '$title' failed to compile: $result" fi done } step_pipelines() { log "Importing pipelines from $SCRIPT_DIR/pipelines/*.json (idempotent)..." local existing existing="$(gcurl GET /system/pipelines/pipeline)" local f title id for f in "$SCRIPT_DIR"/pipelines/*.json; do title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")" id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((p['id'] for p in d if p['title']=='$title'),''))")" if [ -n "$id" ]; then skip "pipeline '$title' already exists ($id)" if [ "$title" = "Network Equipment Parsing" ]; then NETWORK_PIPELINE_ID="$id"; fi if [ "$title" = "Servers Parsing" ]; then SERVERS_PIPELINE_ID="$id"; fi continue fi result="$(gcurl POST /system/pipelines/pipeline "$(cat "$f")")" id="$(echo "$result" | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")" [ -n "$id" ] || die "pipeline '$title' failed to create: $result" ok "created pipeline '$title' ($id)" if [ "$title" = "Network Equipment Parsing" ]; then NETWORK_PIPELINE_ID="$id"; fi if [ "$title" = "Servers Parsing" ]; then SERVERS_PIPELINE_ID="$id"; fi done } step_streams() { log "Creating streams from $SCRIPT_DIR/streams/*.json (idempotent)..." local index_set_id existing index_set_id="$(gcurl GET /system/indices/index_sets | python3 -c "import json,sys;d=json.load(sys.stdin);print(next(s['id'] for s in d['index_sets'] if s.get('default')))")" existing="$(gcurl GET /streams)" # $2.. are extra `sed -e` substitution expressions applied on top of the # index-set-id one, so each template can carry however many input-id # placeholders it needs. render_and_create() { local tmpl="$1" title="$2"; shift 2 local id id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((s['id'] for s in d['streams'] if s['title']=='$title'),''))")" if [ -n "$id" ]; then skip "stream '$title' already exists ($id)" echo "$id" return fi local sed_args=(-e "s/__DEFAULT_INDEX_SET_ID__/$index_set_id/") local expr for expr in "$@"; do sed_args+=(-e "$expr"); done local body body="$(sed "${sed_args[@]}" "$tmpl")" id="$(gcurl POST /streams "$body" | python3 -c "import json,sys;print(json.load(sys.stdin)['stream_id'])")" ok "created stream '$title' ($id)" gcurl POST "/streams/$id/resume" "" >/dev/null echo "$id" } NETWORK_STREAM_ID="$(render_and_create "$SCRIPT_DIR/streams/stream1_network.json" "Network Equipment" \ "s/__NETWORK_INPUT_ID_514__/$NETWORK_INPUT_ID_514/" \ "s/__NETWORK_INPUT_ID_1514__/$NETWORK_INPUT_ID_1514/")" SERVERS_STREAM_ID="$(render_and_create "$SCRIPT_DIR/streams/stream2_servers.json" "Servers" \ "s/__SERVERS_INPUT_ID__/$SERVERS_INPUT_ID/")" } step_connect_pipelines() { log "Connecting pipelines to streams..." gcurl POST /system/pipelines/connections/to_stream \ "{\"stream_id\":\"$NETWORK_STREAM_ID\",\"pipeline_ids\":[\"$NETWORK_PIPELINE_ID\"]}" >/dev/null gcurl POST /system/pipelines/connections/to_stream \ "{\"stream_id\":\"$SERVERS_STREAM_ID\",\"pipeline_ids\":[\"$SERVERS_PIPELINE_ID\"]}" >/dev/null ok "Pipelines connected to streams" } step_firewall() { cp "$SCRIPT_DIR/nftables.conf" /etc/nftables.conf systemctl enable nftables >/dev/null 2>&1 || true nft -f /etc/nftables.conf ok "nftables applied (only 9000/tcp, 514+1514/udp, 5140/udp open)" } step_dashboard() { # Three focused dashboards instead of one combined view - each targets a # different reader (landing/alerts overview, network-equipment-only, # servers-only) so opening Graylog goes straight to what's relevant # instead of one big mixed-stream page. local existing_titles existing_titles="$(gcurl GET /views | python3 -c "import json,sys;print('\n'.join(v['title'] for v in json.load(sys.stdin)['views']))")" local search_file view_file title id search_id body for search_file in "$SCRIPT_DIR"/dashboards/search_*.json; do view_file="${search_file/search_/view_}" [ -f "$view_file" ] || die "missing $view_file for $search_file" title="$(python3 -c "import json;print(json.load(open('$view_file'))['entity']['title'])")" if echo "$existing_titles" | grep -qx "$title"; then skip "dashboard '$title' already exists" continue fi body="$(sed -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/g" -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/g" "$search_file")" search_id="$(gcurl POST /views/search "$body" | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")" [ -n "$search_id" ] || die "search creation failed for dashboard '$title'" body="$(sed -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/g" -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/g" -e "s/__SEARCH_ID__/$search_id/" "$view_file")" id="$(gcurl POST /views "$body" | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")" [ -n "$id" ] || die "dashboard '$title' failed to create" ok "created dashboard '$title' ($id)" done } step_alerts() { if [ -z "$DISCORD_WEBHOOK_URL" ]; then skip "Discord notification and alerts (DISCORD_WEBHOOK_URL not set)" return fi if [ ! -f "$SCRIPT_DIR/alerts/notification_discord.json" ]; then skip "Alerts (no alerts/notification_discord.json in this checkout)" return fi log "Creating Discord notification (idempotent)..." local existing_notifs notif_id existing_notifs="$(gcurl GET /events/notifications)" notif_id="$(echo "$existing_notifs" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((n['id'] for n in d['notifications'] if n['title']=='Discord - Critical Alerts'),''))")" if [ -n "$notif_id" ]; then skip "Discord notification already exists ($notif_id)" else local webhook_slack body webhook_slack="${DISCORD_WEBHOOK_URL%/}/slack" body="$(sed "s#__DISCORD_WEBHOOK_URL_SLACK__#${webhook_slack}#" "$SCRIPT_DIR/alerts/notification_discord.json")" notif_id="$(gcurl POST /events/notifications "{\"entity\": ${body}, \"share_request\": {\"selected_grantee_capabilities\": {}}}" \ | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")" ok "created Discord notification ($notif_id)" fi log "Creating alert (event) definitions from $SCRIPT_DIR/alerts/*.json (idempotent)..." local existing_defs existing_defs="$(gcurl GET /events/definitions)" local f title id for f in "$SCRIPT_DIR"/alerts/alert*.json; do [ -f "$f" ] || continue title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")" id="$(echo "$existing_defs" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((e['id'] for e in d['event_definitions'] if e['title']=='$title'),''))")" if [ -n "$id" ]; then skip "alert '$title' already exists ($id)" continue fi local body body="$(sed \ -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/" \ -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/" \ -e "s/__DISCORD_NOTIFICATION_ID__/$notif_id/" \ "$f")" id="$(gcurl POST /events/definitions "{\"entity\": ${body}, \"share_request\": {\"selected_grantee_capabilities\": {}}}" \ | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")" [ -n "$id" ] || die "alert '$title' failed to create" gcurl PUT "/events/definitions/$id/schedule" "" >/dev/null ok "created and enabled alert '$title' ($id)" done } # --------------------------------------------------------------------------- main() { require_root step_packages step_disable_ipv6 step_docker_install step_check_max_map_count step_docker_smoke_test step_firewall step_compose_files step_compose_up # Must be a plain statement (not `x=$(...)`) so a failure here truly # aborts the script - gcurl() is always called from inside command # substitutions downstream, where `die`'s `exit` would only kill a subshell. resolve_admin_password step_index_retention step_inputs step_pipeline_rules step_pipelines step_streams step_connect_pipelines step_dashboard step_alerts echo >&2 echo "${C_GREEN}==================================================${C_RESET}" >&2 echo "${C_GREEN} DONE - Graylog is up${C_RESET}" >&2 echo "${C_GREEN}==================================================${C_RESET}" >&2 echo " Web UI: $GRAYLOG_EXTERNAL_URI" >&2 echo " Config: $INSTALL_DIR/docker-compose.yml and $INSTALL_DIR/.env" >&2 if [ -f "$INSTALL_DIR/.admin_credentials_ONE_TIME" ]; then echo " Admin credentials: $INSTALL_DIR/.admin_credentials_ONE_TIME (read once, then delete)" >&2 fi echo "${C_GREEN}==================================================${C_RESET}" >&2 } main "$@"