From 510ca37e488595f84a73d14e9d5a78eb8acab457 Mon Sep 17 00:00:00 2001 From: byrsapty Date: Wed, 22 Jul 2026 21:31:08 +0300 Subject: [PATCH] Add flood alerts, session correlation, focused dashboards, and CI deploy workflow - Two flood-detection alerts (per-source message volume, calibrated live against real traffic) grouped by gl2_remote_ip - Session correlation: accelppp_interface fallback tagging plus radius_session_id/calling_station_id/radius_username extraction, so a subscriber's full session lifecycle is searchable by one key - Replace the single combined dashboard with three focused ones (Overview & Alerts, Network Equipment, Servers & Sessions) - Propagate GRAYLOG_ROOT_TIMEZONE and IP-in-alerts fixes into the reusable install script and templates - Add a Forgejo Actions workflow (manual trigger) that re-runs install-graylog.sh on a self-hosted runner living in the container, automating the deploy step this project has done by hand all along Co-Authored-By: Claude Sonnet 5 --- .forgejo/workflows/deploy.yml | 18 ++ README.md | 155 ++++++++++++++---- README.uk.md | 153 ++++++++++++++--- alerts/alert5_server_message_flood.json | 21 +++ alerts/alert6_network_message_flood.json | 21 +++ alerts/notification_discord.json | 2 +- .../{search.json => search_network.json} | 36 ++-- dashboards/search_overview.json | 70 ++++++++ dashboards/search_servers.json | 78 +++++++++ dashboards/{view.json => view_network.json} | 83 +++------- dashboards/view_overview.json | 139 ++++++++++++++++ dashboards/view_servers.json | 147 +++++++++++++++++ install-graylog.sh | 44 ++--- pipelines/pipeline2_servers.json | 2 +- rules/rule11_accelppp_radius_accounting.json | 4 +- ...rule20_accelppp_radius_access_request.json | 5 + rules/rule21_accelppp_interface_tag.json | 5 + 17 files changed, 822 insertions(+), 161 deletions(-) create mode 100644 .forgejo/workflows/deploy.yml create mode 100644 alerts/alert5_server_message_flood.json create mode 100644 alerts/alert6_network_message_flood.json rename dashboards/{search.json => search_network.json} (67%) create mode 100644 dashboards/search_overview.json create mode 100644 dashboards/search_servers.json rename dashboards/{view.json => view_network.json} (58%) create mode 100644 dashboards/view_overview.json create mode 100644 dashboards/view_servers.json create mode 100644 rules/rule20_accelppp_radius_access_request.json create mode 100644 rules/rule21_accelppp_interface_tag.json diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml new file mode 100644 index 0000000..5088caf --- /dev/null +++ b/.forgejo/workflows/deploy.yml @@ -0,0 +1,18 @@ +name: Deploy Graylog config +on: + workflow_dispatch: + +jobs: + deploy: + runs-on: self-hosted + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + + - name: Run install-graylog.sh (idempotent - only applies what changed) + env: + GRAYLOG_EXTERNAL_URI: http://93.171.241.5:9000/ + GRAYLOG_ADMIN_PASSWORD: ${{ secrets.GRAYLOG_ADMIN_PASSWORD }} + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + run: | + chmod +x install-graylog.sh + ./install-graylog.sh diff --git a/README.md b/README.md index 4a3d4de..4e6c318 100644 --- a/README.md +++ b/README.md @@ -210,34 +210,56 @@ instead of passing flags. | `--rootfs-storage` | `EX-Ceph` | container disk storage | | `--discord-webhook` | *(none)* | passed through as `DISCORD_WEBHOOK_URL` to the in-container script | -## Dashboard +## Dashboards -A "Network & RADIUS Monitoring" dashboard is created automatically -(Dashboards → Network & RADIUS Monitoring), with five widgets over a -default 7-day window: +Three focused dashboards are created automatically instead of one combined +view - each is scoped to what one kind of reader actually needs, so opening +Graylog goes straight to something relevant instead of one big page mixing +network gear, servers, and alerts together: -- **Messages Over Time by Stream** - stacked bar, so you can see network - equipment vs. server volume at a glance -- **Vendor Breakdown** - pie chart by the `vendor` field the pipeline - rules set -- **Top Event Types** - table of `event_type` counts -- **Critical Events by Type** - the same, but filtered to - `severity_tag:critical` - i.e. only what the three alerts above care about -- **Top Sources** - which devices/servers are sending the most volume +- **Overview & Alerts** (Dashboards → Overview & Alerts) - the landing + page. Recent Alerts (last 24h, pulled straight from the "All events" + stream so you see the actual fired alerts, not just counts), Message + Volume by Source for the last hour (catches a flood visually before the + flood alerts even fire), Events by Priority (24h), and Critical Events by + Type (24h). +- **Network Equipment** (Dashboards → Network Equipment) - scoped to the + Network Equipment stream only, 7-day window: messages over time by + event type, vendor breakdown (Juniper vs. BDCOM), event type table, top + devices by volume. +- **Servers & Sessions** (Dashboards → Servers & Sessions) - scoped to the + Servers stream only, 7-day window: messages over time by event type, + event type table, top servers by volume (same view the flood alerts are + calibrated against), and a RADIUS accounting status breakdown + (Start/Alive/Stop counts). -It's built via the Views API (`dashboards/search.json` + `dashboards/view.json`) -rather than Graylog's own widget-builder UI - that UI turned out to be -difficult to drive reliably via browser automation (React `combobox` -widgets that don't respond to plain keyboard/click events without also -triggering React's internal state update), while the REST API accepted the -same structure cleanly on the first attempt once the shape was reverse -engineered from an existing dashboard's JSON. If you want to add a widget, -either use the Graylog UI directly (a human using a mouse doesn't hit the -automation issue) and then optionally export the result back into these -two JSON files, or extend `dashboards/search.json`/`view.json` by hand - -each widget needs a matching `search_types` entry (in `search.json`) and -`widgets` + `widget_mapping` + `positions` + `titles.widget` entry (in -`view.json`) sharing the same ID. +Each is built via the Views API (`dashboards/search_.json` + +`dashboards/view_.json` pairs, one pair per dashboard) rather than +Graylog's own widget-builder UI - that UI turned out to be difficult to +drive reliably via browser automation (React `combobox` widgets that don't +respond to plain keyboard/click events without also triggering React's +internal state update), while the REST API accepted the same structure +cleanly once the shape was reverse engineered from an existing dashboard's +JSON. The one part that isn't a plain aggregation pivot - the Recent Alerts +widget, `type: "messages"` instead of `type: "aggregation"` - needed its +own bit of reverse engineering too: the widget-level `sort` field on a +message-list widget must be `[]`, not an object with a `field`/`order` +pair, or Graylog rejects it with a Jackson polymorphism error +(`missing type id property 'type'` - the sort DTO for message widgets +doesn't have any registered subtypes in this Graylog version at all). + +If you want to add a widget, either use the Graylog UI directly (a human +using a mouse doesn't hit the automation issue) and then optionally export +the result back into these JSON files, or extend a `search_.json`/ +`view_.json` pair by hand - each widget needs a matching +`search_types` entry (in `search_.json`) and `widgets` + +`widget_mapping` + `positions` + `titles.widget` entry (in +`view_.json`) sharing the same ID. `step_dashboard()` in +`install-graylog.sh` picks up any `search_*.json`/`view_*.json` pair +automatically (matched by filename), so a new pair just needs to exist in +the `dashboards/` directory - no script changes required. `__NETWORK_STREAM_ID__` +and `__SERVERS_STREAM_ID__` placeholders are substituted the same way the +alert templates already do it. ## Known environment quirks this script works around @@ -318,9 +340,9 @@ each widget needs a matching `search_types` entry (in `search.json`) and ## Alerting and Discord notifications -Three CRITICAL-only alerts are wired up out of the box (everything else stays -quiet on purpose - routine auth failures, single dropped-session events, etc. -are parsed and searchable but never page anyone): +Six alerts are wired up out of the box (everything else stays quiet on +purpose - routine auth failures, single dropped-session events, etc. are +parsed and searchable but never page anyone): | Alert | Fires on | Priority | |---|---|---| @@ -328,11 +350,44 @@ are parsed and searchable but never page anyone): | conntrack table full (packet loss) | `nf_conntrack: table full, dropping packet` (standard Linux kernel message - active, ongoing packet loss) | High | | Unrecognized critical-severity syslog | Any message (any vendor, either stream) with RFC5424/3164 syslog severity Emergency/Alert/Critical (0-2) that no specific pipeline rule already classified | Medium | | Juniper chassis hardware alarm | `CHASSISD_SNMP_TRAP`/`CHASSISD_SNMP_TRAP6` (over temperature, fan, power supply, etc.) - confirmed live against a real `VC.MYRONIVKA` chassis | High | +| Abnormal message volume from one server | A single server in the Servers stream sends more than 150,000 messages in a 10-minute window - see "Message-volume (flood) alerts" below | Medium | +| Abnormal syslog volume from network equipment | A single device in the Network Equipment stream sends more than 500 messages in a 5-minute window - see "Message-volume (flood) alerts" below | Medium | That third one is the "universal network equipment problem" catch-all: it doesn't depend on knowing any vendor's specific message format, just the standard syslog severity level every reasonable device already sends. +### Message-volume (flood) alerts + +The last two alerts protect against a single misbehaving source silently +filling the retention window's disk budget - a log loop, a retry storm, or +a debug-level setting left on by accident. They group by `gl2_remote_ip` +(aggregation-v1, `count() > threshold`), so each *individual* source is +compared against its own volume, not the whole stream's total. + +The thresholds are not guessed - they were calibrated live on 2026-07-22 +against real traffic, via a Views API pivot search grouped by +`gl2_remote_ip`: +- The one active accel-ppp/RADIUS server was steadily sending **~4,600-4,800 + messages per 10 minutes** (~278k/hour) under normal load. The Servers + stream threshold (150,000/10min) gives roughly 3x headroom above that. +- The one active network device was steadily sending **~30-60 messages per + 5 minutes** (~360/hour). The Network Equipment stream threshold + (500/5min) gives roughly 10x headroom above that. + +These are starting points based on partial rollout (1 server + 1 device +active at calibration time). Revisit both thresholds once more of the +planned ~10-15 servers and ~10-20 switches/OLTs are sending real traffic - +what looks like 3x headroom today could be too tight or too loose once +every server's individual baseline is known. Check current per-source +volume any time with a query like: +``` +gl2_remote_ip: +``` +over a fixed time range in the Search page, or reuse the same pivot-search +approach (grouped by `gl2_remote_ip`, `count()` series) via the Views API +if you want exact numbers instead of eyeballing a graph. + Also parsed (searchable, but not alerted on since they're routine/expected volume, not incidents by themselves): - accel-ppp: PPP authentication failed (`ppp_auth.c`) @@ -340,6 +395,43 @@ volume, not incidents by themselves): `Auth: Login incorrect: [user] (from client X port P)` (default `auth_log` format) +## Session correlation (accel-ppp subscriber sessions) + +Every accel-ppp log line for a given subscriber session - RADIUS +Access-Request (auth attempt), Accounting-Request (start/interim/stop), +DHCP discover/offer/request/ack, ipoe session create/start/finish/terminate +- gets tagged with the same `accelppp_interface` field (the `vlanNNNN.NNN` +interface name accel-ppp itself uses per subscriber). This works even for +message types with no other structured fields at all, via a fallback rule +(`accelppp_interface_tag`) that only tags lines no more specific rule +already classified. + +To see a subscriber's full session lifecycle in one query, search: +``` +accelppp_interface:"vlan1779.124" +``` +sorted by time (default). This surfaces the DHCP handshake, the RADIUS +auth/accounting exchange, and the eventual termination as one chronological +list, instead of grep-ing for the interface name across raw text. + +RADIUS Access-Request and Accounting-Request lines additionally get three +richer correlation fields extracted directly from the RADIUS AVPs: +- `radius_session_id` - accel-ppp's `Acct-Session-Id`, stable for the + entire session +- `calling_station_id` - the subscriber's MAC address +- `radius_username` - the subscriber's login (accel-ppp's `User-Name`, + format `:` in this deployment) + +These are useful when starting from a support ticket that has a MAC address +or username but not the interface name, e.g.: +``` +calling_station_id:"48:8f:5a:a4:f9:ba" +``` +Confirmed live on 2026-07-22 against real EX-NAS-1-1 traffic: a single +`accelppp_interface` value correctly tied together a DHCPv4 Ack, a DHCPv4 +Request, a RADIUS Accounting-Response, and a RADIUS Accounting-Request, all +belonging to the same subscriber session. + To wire up Discord, pass `--discord-webhook` (or set `DISCORD_WEBHOOK_URL`) when running `create-graylog-lxc.sh`. Under the hood this uses Graylog's built-in **Slack** notification type pointed at @@ -350,9 +442,16 @@ matched message: ``` *${event_definition_title}* ${event_definition_description} +${event.message} ${if backlog}${foreach backlog message}• `${message.source}` (IP: ${message.fields.gl2_remote_ip}): ${message.message} ${end}${end} ``` +`${event.message}` is Graylog's own auto-generated event summary - for +plain critical alerts it just duplicates the title, but for the two flood +alerts (grouped by `gl2_remote_ip`) this is where the specific source IP +and the actual `count()` value show up, e.g. +`WARNING: ...: 93.171.243.4 - count()=278474.0`. + `gl2_remote_ip` is a field Graylog attaches automatically to every message based on the actual UDP packet's source address, regardless of what hostname the device itself claims in the syslog `source` field. diff --git a/README.uk.md b/README.uk.md index 64d5e07..3c994cc 100644 --- a/README.uk.md +++ b/README.uk.md @@ -214,35 +214,55 @@ Graylog під кожен пристрій не потрібно — inputs і | `--rootfs-storage` | `EX-Ceph` | сховище для диска контейнера | | `--discord-webhook` | *(немає)* | передається як `DISCORD_WEBHOOK_URL` у скрипт всередині контейнера | -## Дашборд +## Дашборди -Дашборд "Network & RADIUS Monitoring" створюється автоматично (Dashboards -→ Network & RADIUS Monitoring), з п'ятьма віджетами за замовчуванням на -7-денному вікні: +Замість одного комбінованого дашборду автоматично створюються три +фокусовані — кожен під конкретного читача, тож відкривши Graylog, одразу +бачиш релевантне, а не одну велику сторінку зі змішаними мережею, +серверами й алертами: -- **Messages Over Time by Stream** — накопичувальна стовпчикова діаграма, - щоб одним поглядом бачити обсяг мережевого обладнання vs. серверів -- **Vendor Breakdown** — кругова діаграма за полем `vendor`, яке - проставляють pipeline rules -- **Top Event Types** — таблиця з підрахунком по `event_type` -- **Critical Events by Type** — те саме, але відфільтроване по - `severity_tag:critical` — тобто саме те, чим переймаються три алерти вище -- **Top Sources** — які пристрої/сервери генерують найбільше обсягу +- **Overview & Alerts** (Dashboards → Overview & Alerts) — стартова + сторінка. Recent Alerts (останні 24г, витягнуто прямо зі стріму "All + events", тож видно самі спрацьовані алерти, а не лише лічильники), + Message Volume by Source за останню годину (ловить флуд візуально ще до + того, як спрацюють flood-алерти), Events by Priority (24г) та Critical + Events by Type (24г). +- **Network Equipment** (Dashboards → Network Equipment) — тільки стрім + Network Equipment, 7-денне вікно: обсяг у часі по event_type, розбивка + по вендору (Juniper vs. BDCOM), таблиця event type, топ пристроїв за + обсягом. +- **Servers & Sessions** (Dashboards → Servers & Sessions) — тільки стрім + Servers, 7-денне вікно: обсяг у часі по event_type, таблиця event type, + топ серверів за обсягом (той самий зріз, під який каліброві + flood-алерти), і розбивка RADIUS accounting статусів (Start/Alive/Stop). -Побудований через Views API (`dashboards/search.json` + `dashboards/view.json`), -а не через власний конструктор віджетів Graylog у браузері — цей -конструктор виявився складно керованим надійно через браузерну -автоматизацію (React `combobox`-віджети, які не реагують на прості -keyboard/click-події без одночасного тригера внутрішнього React-стану), -тоді як REST API прийняв ту саму структуру чисто з першої спроби, щойно -формат був реконструйований із JSON існуючого дашборду. Якщо хочете додати -віджет — або скористайтесь Graylog UI напряму (людина з мишкою не -натикається на проблему автоматизації), а потім за бажанням перенесіть -результат назад у ці два JSON-файли, або розширте -`dashboards/search.json`/`view.json` вручну — кожен віджет потребує -відповідного запису в `search_types` (у `search.json`) та +Кожен побудований через Views API (пари `dashboards/search_<назва>.json` + +`dashboards/view_<назва>.json`, по одній парі на дашборд), а не через +власний конструктор віджетів Graylog у браузері — цей конструктор виявився +складно керованим надійно через браузерну автоматизацію (React +`combobox`-віджети, які не реагують на прості keyboard/click-події без +одночасного тригера внутрішнього React-стану), тоді як REST API прийняв ту +саму структуру чисто, щойно формат був реконструйований із JSON існуючого +дашборду. Єдина частина, що не є простим aggregation pivot — віджет Recent +Alerts, `type: "messages"` замість `type: "aggregation"` — потребувала +власної реконструкції: поле `sort` на рівні віджета для message-list +віджету має бути `[]`, а не об'єктом з парою `field`/`order`, інакше +Graylog відхиляє його з помилкою Jackson-поліморфізму +(`missing type id property 'type'` — DTO сортування для message-віджетів +взагалі не має зареєстрованих підтипів у цій версії Graylog). + +Якщо хочете додати віджет — або скористайтесь Graylog UI напряму (людина з +мишкою не натикається на проблему автоматизації), а потім за бажанням +перенесіть результат назад у ці JSON-файли, або розширте пару +`search_<назва>.json`/`view_<назва>.json` вручну — кожен віджет потребує +відповідного запису в `search_types` (у `search_<назва>.json`) та `widgets` + `widget_mapping` + `positions` + `titles.widget` (у -`view.json`) з однаковим ID. +`view_<назва>.json`) з однаковим ID. `step_dashboard()` в +`install-graylog.sh` сам підхоплює будь-яку пару +`search_*.json`/`view_*.json` (за збігом імені файлу), тож нова пара +просто має існувати в директорії `dashboards/` — жодних змін у скрипті не +потрібно. Плейсхолдери `__NETWORK_STREAM_ID__` та `__SERVERS_STREAM_ID__` +підставляються так само, як і в шаблонах алертів. ## Особливості середовища, які скрипт обходить @@ -336,8 +356,8 @@ keyboard/click-події без одночасного тригера внут ## Алерти та Discord-нотифікації -Три алерти працюють одразу з коробки, і всі — тільки на критичні події -(рутинні auth-fail, поодинокі розриви сесій тощо парсяться й доступні для +Шість алертів працюють одразу з коробки (все інше навмисно мовчить — +рутинні auth-fail, поодинокі розриви сесій тощо парсяться й доступні для пошуку, але нікого не турбують сповіщенням): | Алерт | Спрацьовує на | Пріоритет | @@ -346,12 +366,45 @@ keyboard/click-події без одночасного тригера внут | conntrack table full (packet loss) | `nf_conntrack: table full, dropping packet` (стандартне повідомлення ядра Linux — активна втрата пакетів прямо зараз) | High | | Unrecognized critical-severity syslog | Будь-яке повідомлення (будь-який вендор, будь-який стрім) із syslog-severity Emergency/Alert/Critical (0-2) за RFC5424/3164, яке не класифікувало жодне спеціальне правило | Medium | | Juniper chassis hardware alarm | `CHASSISD_SNMP_TRAP`/`CHASSISD_SNMP_TRAP6` (перегрів, вентилятор, блок живлення тощо) — перевірено наживо на реальному шасі `VC.MYRONIVKA` | High | +| Аномальний обсяг повідомлень від одного сервера | Один сервер у стрімі Servers шле понад 150 000 повідомлень за 10-хвилинне вікно — див. "Алерти на обсяг (flood)" нижче | Medium | +| Аномальний обсяг syslog від мережевого обладнання | Один пристрій у стрімі Network Equipment шле понад 500 повідомлень за 5-хвилинне вікно — див. "Алерти на обсяг (flood)" нижче | Medium | Третій алерт і є тим самим "універсальним" покриттям проблем мережевого обладнання: він не залежить від знання формату повідомлень конкретного вендора — лише від стандартного рівня severity syslog, який шле будь-який притомний пристрій. +### Алерти на обсяг (flood) + +Останні два алерти захищають від ситуації, коли одне джерело тихо забиває +диск у вікні retention — цикл у логах, шторм ретраїв, чи випадково +залишений debug-рівень. Вони групуються за `gl2_remote_ip` +(`aggregation-v1`, `count() > поріг`), тож кожне джерело порівнюється зі +своїм власним обсягом, а не із сумою по всьому стріму. + +Пороги не вигадані — вони каліброві наживо 2026-07-22 на реальному +трафіку через pivot-пошук Views API, згрупований за `gl2_remote_ip`: +- Один активний accel-ppp/RADIUS сервер стабільно слав **~4 600-4 800 + повідомлень за 10 хвилин** (~278 тис/год) за нормального навантаження. + Поріг для стріму Servers (150 000/10хв) дає приблизно 3x запасу над цим. +- Один активний мережевий пристрій стабільно слав **~30-60 повідомлень за + 5 хвилин** (~360/год). Поріг для Network Equipment (500/5хв) дає + приблизно 10x запасу над цим. + +Це стартові значення на основі часткового розгортання (1 сервер + 1 +пристрій були активні на момент калібрування). Переглядайте обидва пороги +по мірі підключення решти запланованих ~10-15 серверів і ~10-20 +комутаторів/OLT — те, що сьогодні виглядає як 3x запасу, може виявитись +затісним або занадто вільним, коли буде відомий власний baseline кожного +сервера. Перевірити поточний обсяг по джерелу можна будь-коли запитом +на кшталт: +``` +gl2_remote_ip: +``` +за фіксований проміжок часу на сторінці Search, або тим самим +pivot-пошуком (групування за `gl2_remote_ip`, серія `count()`) через Views +API, якщо потрібні точні числа замість погляду на графік. + Також парситься (доступне для пошуку, але без алерту — це рутинний обсяг, а не інцидент сам по собі): - accel-ppp: PPP authentication failed (`ppp_auth.c`) @@ -359,6 +412,45 @@ keyboard/click-події без одночасного тригера внут `Auth: Login incorrect: [user] (from client X port P)` (типовий формат `auth_log`) +## Кореляція сесій (абонентські сесії accel-ppp) + +Кожен рядок логу accel-ppp для конкретної сесії абонента — RADIUS +Access-Request (спроба авторизації), Accounting-Request +(start/interim/stop), DHCP discover/offer/request/ack, ipoe session +create/start/finish/terminate — тепер позначається тим самим полем +`accelppp_interface` (ім'я інтерфейсу `vlanNNNN.NNN`, яке сам accel-ppp +використовує для кожного абонента). Це працює навіть для типів +повідомлень без жодних інших структурованих полів — завдяки fallback +правилу (`accelppp_interface_tag`), яке позначає лише ті рядки, які ще не +класифікувало жодне спеціальне правило. + +Щоб побачити повний життєвий цикл сесії абонента одним запитом, шукайте: +``` +accelppp_interface:"vlan1779.124" +``` +відсортовано за часом (за замовчуванням). Це показує DHCP-хендшейк, +RADIUS auth/accounting обмін і кінцеве завершення сесії як один +хронологічний список — замість пошуку імені інтерфейсу вручну по сирому +тексту. + +Рядки RADIUS Access-Request та Accounting-Request додатково отримують три +багатших поля кореляції, витягнуті прямо з RADIUS AVP: +- `radius_session_id` — `Acct-Session-Id` accel-ppp, стабільний для всієї + сесії +- `calling_station_id` — MAC-адреса абонента +- `radius_username` — логін абонента (`User-Name` accel-ppp, формат + `:` у цьому розгортанні) + +Це корисно, коли відправна точка — тікет з MAC-адресою чи логіном, а не +ім'ям інтерфейсу, наприклад: +``` +calling_station_id:"48:8f:5a:a4:f9:ba" +``` +Перевірено наживо 2026-07-22 на реальному трафіку EX-NAS-1-1: одне +значення `accelppp_interface` коректно об'єднало DHCPv4 Ack, DHCPv4 +Request, RADIUS Accounting-Response та RADIUS Accounting-Request, що +належали одній і тій самій сесії абонента. + Щоб підключити Discord, передайте `--discord-webhook` (або встановіть `DISCORD_WEBHOOK_URL`) при запуску `create-graylog-lxc.sh`. Під капотом використовується вбудований тип нотифікації Graylog **Slack**, спрямований @@ -369,9 +461,16 @@ keyboard/click-події без одночасного тригера внут ``` *${event_definition_title}* ${event_definition_description} +${event.message} ${if backlog}${foreach backlog message}• `${message.source}` (IP: ${message.fields.gl2_remote_ip}): ${message.message} ${end}${end} ``` +Рядок `${event.message}` показує автоматично згенероване Graylog резюме +події — для звичайних critical-алертів це просто дублює заголовок, а для +двох flood-алертів (групованих за `gl2_remote_ip`) саме тут з'являється +конкретна IP-адреса джерела і фактичне значення `count()`, наприклад +`WARNING: ...: 93.171.243.4 - count()=278474.0`. + `gl2_remote_ip` — поле, яке Graylog проставляє автоматично для кожного повідомлення на основі реальної адреси відправника UDP-пакета, незалежно від того, яке ім'я хоста заявляє сам пристрій у полі syslog `source`. diff --git a/alerts/alert5_server_message_flood.json b/alerts/alert5_server_message_flood.json new file mode 100644 index 0000000..58b71c8 --- /dev/null +++ b/alerts/alert5_server_message_flood.json @@ -0,0 +1,21 @@ +{ + "title": "WARNING: abnormal message volume from one server", + "description": "A single server is sending far more syslog messages than its established baseline - possible log loop, misconfiguration, or a genuine incident generating excessive events. Left unchecked this can fill the retention window's disk budget early. Threshold calibrated live on 2026-07-22 against real accel-ppp/RADIUS traffic: the one active NAS server was steadily sending ~4,600-4,800 msgs/10min (~278k/hour) under totally normal load, so the threshold is set to roughly 3x that baseline.", + "priority": 2, + "alert": true, + "config": { + "type": "aggregation-v1", + "query": "*", + "streams": ["__SERVERS_STREAM_ID__"], + "group_by": ["gl2_remote_ip"], + "series": [{"type": "count", "id": "count-", "field": null}], + "conditions": {"expression": {"expr": ">", "left": {"expr": "number-ref", "ref": "count-"}, "right": {"expr": "number", "value": 150000.0}}}, + "search_within_ms": 600000, + "execute_every_ms": 600000, + "event_limit": 50 + }, + "field_spec": {}, + "key_spec": [], + "notification_settings": {"grace_period_ms": 1800000, "backlog_size": 5}, + "notifications": [{"notification_id": "__DISCORD_NOTIFICATION_ID__"}] +} diff --git a/alerts/alert6_network_message_flood.json b/alerts/alert6_network_message_flood.json new file mode 100644 index 0000000..acd6253 --- /dev/null +++ b/alerts/alert6_network_message_flood.json @@ -0,0 +1,21 @@ +{ + "title": "WARNING: abnormal syslog volume from network equipment", + "description": "A single switch/OLT/router is sending far more syslog messages than normal - often a symptom of interface flapping, an ARP/MAC-move loop, or a spanning-tree issue. Threshold calibrated live on 2026-07-22 against real network equipment traffic: the observed device was steadily sending ~30-60 msgs/5min (~360/hour) under normal conditions, so the threshold gives roughly 10x headroom above that baseline.", + "priority": 2, + "alert": true, + "config": { + "type": "aggregation-v1", + "query": "*", + "streams": ["__NETWORK_STREAM_ID__"], + "group_by": ["gl2_remote_ip"], + "series": [{"type": "count", "id": "count-", "field": null}], + "conditions": {"expression": {"expr": ">", "left": {"expr": "number-ref", "ref": "count-"}, "right": {"expr": "number", "value": 500.0}}}, + "search_within_ms": 300000, + "execute_every_ms": 300000, + "event_limit": 50 + }, + "field_spec": {}, + "key_spec": [], + "notification_settings": {"grace_period_ms": 900000, "backlog_size": 5}, + "notifications": [{"notification_id": "__DISCORD_NOTIFICATION_ID__"}] +} diff --git a/alerts/notification_discord.json b/alerts/notification_discord.json index 1a478ae..5c74741 100644 --- a/alerts/notification_discord.json +++ b/alerts/notification_discord.json @@ -6,7 +6,7 @@ "color": "#FF0000", "webhook_url": "__DISCORD_WEBHOOK_URL_SLACK__", "channel": "#alerts", - "custom_message": "*${event_definition_title}*\n${event_definition_description}\n${if backlog}${foreach backlog message}• `${message.source}` (IP: ${message.fields.gl2_remote_ip}): ${message.message}\n${end}${end}", + "custom_message": "*${event_definition_title}*\n${event_definition_description}\n${event.message}\n${if backlog}${foreach backlog message}• `${message.source}` (IP: ${message.fields.gl2_remote_ip}): ${message.message}\n${end}${end}", "user_name": "Graylog", "notify_channel": false, "notify_here": false, diff --git a/dashboards/search.json b/dashboards/search_network.json similarity index 67% rename from dashboards/search.json rename to dashboards/search_network.json index 7a867ef..bfe341e 100644 --- a/dashboards/search.json +++ b/dashboards/search_network.json @@ -1,35 +1,35 @@ { "queries": [ { - "id": "89d6c0f3-3d35-42ed-9ad3-14cd7daff7c1", + "id": "a2000000-0000-0000-0000-000000000001", "timerange": {"type": "relative", "range": 604800}, "filter": null, "filters": [], "query": {"type": "elasticsearch", "query_string": ""}, "search_types": [ { - "id": "b5613b69-ccc8-40a6-a7c6-8c9b9b909abf", + "id": "a2000000-0000-0000-0000-000000000011", "type": "pivot", "name": "chart", "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "series": [{"type": "count", "id": "Message count", "field": null}], "sort": [], "rollup": false, "row_groups": [{"type": "time", "fields": ["timestamp"], "interval": {"type": "auto", "scaling": 1.0}}], - "column_groups": [{"type": "values", "fields": ["streams"], "limit": 15, "skip_empty_values": false}], + "column_groups": [{"type": "values", "fields": ["event_type"], "limit": 10, "skip_empty_values": false}], "filter": null, "filters": [] }, { - "id": "01f7bdc5-14f0-4221-9bbf-ba3bea6b308c", + "id": "a2000000-0000-0000-0000-000000000012", "type": "pivot", "name": "chart", "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "series": [{"type": "count", "id": "Message count", "field": null}], "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], @@ -40,12 +40,12 @@ "filters": [] }, { - "id": "1295ec49-8bb6-42d2-87ba-adad9ed43e73", + "id": "a2000000-0000-0000-0000-000000000013", "type": "pivot", "name": "chart", "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "series": [{"type": "count", "id": "Message count", "field": null}], "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], @@ -56,28 +56,12 @@ "filters": [] }, { - "id": "d9551aab-be85-43b6-a205-02a8a6434401", - "type": "pivot", - "name": "chart", - "timerange": {"type": "relative", "range": 604800}, - "query": {"type": "elasticsearch", "query_string": "severity_tag:critical"}, - "streams": [], - "stream_categories": [], - "series": [{"type": "count", "id": "Message count", "field": null}], - "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], - "rollup": true, - "row_groups": [{"type": "values", "fields": ["event_type"], "limit": 15, "skip_empty_values": true}], - "column_groups": [], - "filter": null, - "filters": [] - }, - { - "id": "8694431d-0f1a-4162-910e-de6d3fbe0b3e", + "id": "a2000000-0000-0000-0000-000000000014", "type": "pivot", "name": "chart", "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "series": [{"type": "count", "id": "Message count", "field": null}], "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], diff --git a/dashboards/search_overview.json b/dashboards/search_overview.json new file mode 100644 index 0000000..2be4866 --- /dev/null +++ b/dashboards/search_overview.json @@ -0,0 +1,70 @@ +{ + "queries": [ + { + "id": "a1000000-0000-0000-0000-000000000001", + "timerange": {"type": "relative", "range": 86400}, + "filter": null, + "filters": [], + "query": {"type": "elasticsearch", "query_string": ""}, + "search_types": [ + { + "id": "a1000000-0000-0000-0000-000000000011", + "type": "messages", + "streams": ["000000000000000000000002"], + "limit": 25, + "offset": 0, + "sort": [{"field": "timestamp", "order": "DESC"}] + }, + { + "id": "a1000000-0000-0000-0000-000000000012", + "type": "pivot", + "name": "chart", + "timerange": {"type": "relative", "range": 3600}, + "query": null, + "streams": [], + "stream_categories": [], + "series": [{"type": "count", "id": "Message count", "field": null}], + "sort": [], + "rollup": false, + "row_groups": [{"type": "time", "fields": ["timestamp"], "interval": {"type": "auto", "scaling": 1.0}}], + "column_groups": [{"type": "values", "fields": ["gl2_remote_ip"], "limit": 10, "skip_empty_values": false}], + "filter": null, + "filters": [] + }, + { + "id": "a1000000-0000-0000-0000-000000000013", + "type": "pivot", + "name": "chart", + "timerange": {"type": "relative", "range": 86400}, + "query": null, + "streams": ["000000000000000000000002"], + "stream_categories": [], + "series": [{"type": "count", "id": "Message count", "field": null}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "rollup": true, + "row_groups": [{"type": "values", "fields": ["priority"], "limit": 10, "skip_empty_values": true}], + "column_groups": [], + "filter": null, + "filters": [] + }, + { + "id": "a1000000-0000-0000-0000-000000000014", + "type": "pivot", + "name": "chart", + "timerange": {"type": "relative", "range": 86400}, + "query": {"type": "elasticsearch", "query_string": "severity_tag:critical"}, + "streams": [], + "stream_categories": [], + "series": [{"type": "count", "id": "Message count", "field": null}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "rollup": true, + "row_groups": [{"type": "values", "fields": ["event_type"], "limit": 15, "skip_empty_values": true}], + "column_groups": [], + "filter": null, + "filters": [] + } + ] + } + ], + "parameters": [] +} diff --git a/dashboards/search_servers.json b/dashboards/search_servers.json new file mode 100644 index 0000000..e5d51c5 --- /dev/null +++ b/dashboards/search_servers.json @@ -0,0 +1,78 @@ +{ + "queries": [ + { + "id": "a3000000-0000-0000-0000-000000000001", + "timerange": {"type": "relative", "range": 604800}, + "filter": null, + "filters": [], + "query": {"type": "elasticsearch", "query_string": ""}, + "search_types": [ + { + "id": "a3000000-0000-0000-0000-000000000011", + "type": "pivot", + "name": "chart", + "timerange": {"type": "relative", "range": 604800}, + "query": null, + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "series": [{"type": "count", "id": "Message count", "field": null}], + "sort": [], + "rollup": false, + "row_groups": [{"type": "time", "fields": ["timestamp"], "interval": {"type": "auto", "scaling": 1.0}}], + "column_groups": [{"type": "values", "fields": ["event_type"], "limit": 10, "skip_empty_values": false}], + "filter": null, + "filters": [] + }, + { + "id": "a3000000-0000-0000-0000-000000000012", + "type": "pivot", + "name": "chart", + "timerange": {"type": "relative", "range": 604800}, + "query": null, + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "series": [{"type": "count", "id": "Message count", "field": null}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "rollup": true, + "row_groups": [{"type": "values", "fields": ["event_type"], "limit": 15, "skip_empty_values": true}], + "column_groups": [], + "filter": null, + "filters": [] + }, + { + "id": "a3000000-0000-0000-0000-000000000013", + "type": "pivot", + "name": "chart", + "timerange": {"type": "relative", "range": 604800}, + "query": null, + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "series": [{"type": "count", "id": "Message count", "field": null}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "rollup": true, + "row_groups": [{"type": "values", "fields": ["source"], "limit": 15, "skip_empty_values": true}], + "column_groups": [], + "filter": null, + "filters": [] + }, + { + "id": "a3000000-0000-0000-0000-000000000014", + "type": "pivot", + "name": "chart", + "timerange": {"type": "relative", "range": 604800}, + "query": "event_type:radius_accounting", + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "series": [{"type": "count", "id": "Message count", "field": null}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "rollup": true, + "row_groups": [{"type": "values", "fields": ["acct_status_type"], "limit": 10, "skip_empty_values": true}], + "column_groups": [], + "filter": null, + "filters": [] + } + ] + } + ], + "parameters": [] +} diff --git a/dashboards/view.json b/dashboards/view_network.json similarity index 58% rename from dashboards/view.json rename to dashboards/view_network.json index fb3a27f..85ccaac 100644 --- a/dashboards/view.json +++ b/dashboards/view_network.json @@ -1,40 +1,39 @@ { "entity": { "type": "DASHBOARD", - "title": "Network & RADIUS Monitoring", - "summary": "Overview of collected syslog: volume, sources, vendors, event types, and critical issues.", - "description": "Built for the Juniper/BDCOM/accel-ppp/RADIUS log collection pipeline. Default range is 7 days - adjust with the time selector at the top.", + "title": "Network Equipment", + "summary": "Juniper/BDCOM switches, routers and OLTs - vendor breakdown, event types, top devices.", + "description": "Scoped to the Network Equipment stream only. Default range is 7 days.", "search_id": "__SEARCH_ID__", "properties": [], "requires": {}, "state": { - "89d6c0f3-3d35-42ed-9ad3-14cd7daff7c1": { + "a2000000-0000-0000-0000-000000000001": { "selected_fields": null, "static_message_list_id": null, "titles": { - "tab": {"title": "Overview"}, + "tab": {"title": "Network Equipment"}, "widget": { - "b5613b69-ccc8-40a6-a7c6-8c9b9b909abf": "Messages Over Time by Stream", - "01f7bdc5-14f0-4221-9bbf-ba3bea6b308c": "Vendor Breakdown", - "1295ec49-8bb6-42d2-87ba-adad9ed43e73": "Top Event Types", - "d9551aab-be85-43b6-a205-02a8a6434401": "Critical Events by Type", - "8694431d-0f1a-4162-910e-de6d3fbe0b3e": "Top Sources" + "a2000000-0000-0000-0000-000000000011": "Messages Over Time by Event Type", + "a2000000-0000-0000-0000-000000000012": "Vendor Breakdown", + "a2000000-0000-0000-0000-000000000013": "Event Types", + "a2000000-0000-0000-0000-000000000014": "Top Devices" } }, "widgets": [ { - "id": "b5613b69-ccc8-40a6-a7c6-8c9b9b909abf", + "id": "a2000000-0000-0000-0000-000000000011", "type": "aggregation", "filter": null, "filters": [], "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "config": { "row_pivots": [{"fields": ["timestamp"], "type": "time", "config": {"interval": {"type": "auto", "scaling": 1.0}}}], "units": {}, - "column_pivots": [{"fields": ["streams"], "type": "values", "config": {"limit": 15}}], + "column_pivots": [{"fields": ["event_type"], "type": "values", "config": {"limit": 10}}], "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], "sort": [], "visualization": "bar", @@ -43,19 +42,19 @@ "rollup": false, "event_annotation": false, "row_limit": null, - "column_limit": 15 + "column_limit": 10 }, "description": null, "context": null }, { - "id": "01f7bdc5-14f0-4221-9bbf-ba3bea6b308c", + "id": "a2000000-0000-0000-0000-000000000012", "type": "aggregation", "filter": null, "filters": [], "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "config": { "row_pivots": [{"fields": ["vendor"], "type": "values", "config": {"limit": 10}}], @@ -75,13 +74,13 @@ "context": null }, { - "id": "1295ec49-8bb6-42d2-87ba-adad9ed43e73", + "id": "a2000000-0000-0000-0000-000000000013", "type": "aggregation", "filter": null, "filters": [], "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "config": { "row_pivots": [{"fields": ["event_type"], "type": "values", "config": {"limit": 15}}], @@ -101,39 +100,13 @@ "context": null }, { - "id": "d9551aab-be85-43b6-a205-02a8a6434401", - "type": "aggregation", - "filter": null, - "filters": [], - "timerange": {"type": "relative", "range": 604800}, - "query": "severity_tag:critical", - "streams": [], - "stream_categories": [], - "config": { - "row_pivots": [{"fields": ["event_type"], "type": "values", "config": {"limit": 15}}], - "units": {}, - "column_pivots": [], - "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], - "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], - "visualization": "table", - "visualization_config": {"pinned_columns": [], "show_row_numbers": true}, - "formatting_settings": null, - "rollup": true, - "event_annotation": false, - "row_limit": 15, - "column_limit": null - }, - "description": null, - "context": null - }, - { - "id": "8694431d-0f1a-4162-910e-de6d3fbe0b3e", + "id": "a2000000-0000-0000-0000-000000000014", "type": "aggregation", "filter": null, "filters": [], "timerange": {"type": "relative", "range": 604800}, "query": null, - "streams": [], + "streams": ["__NETWORK_STREAM_ID__"], "stream_categories": [], "config": { "row_pivots": [{"fields": ["source"], "type": "values", "config": {"limit": 15}}], @@ -154,18 +127,16 @@ } ], "widget_mapping": { - "b5613b69-ccc8-40a6-a7c6-8c9b9b909abf": ["b5613b69-ccc8-40a6-a7c6-8c9b9b909abf"], - "01f7bdc5-14f0-4221-9bbf-ba3bea6b308c": ["01f7bdc5-14f0-4221-9bbf-ba3bea6b308c"], - "1295ec49-8bb6-42d2-87ba-adad9ed43e73": ["1295ec49-8bb6-42d2-87ba-adad9ed43e73"], - "d9551aab-be85-43b6-a205-02a8a6434401": ["d9551aab-be85-43b6-a205-02a8a6434401"], - "8694431d-0f1a-4162-910e-de6d3fbe0b3e": ["8694431d-0f1a-4162-910e-de6d3fbe0b3e"] + "a2000000-0000-0000-0000-000000000011": ["a2000000-0000-0000-0000-000000000011"], + "a2000000-0000-0000-0000-000000000012": ["a2000000-0000-0000-0000-000000000012"], + "a2000000-0000-0000-0000-000000000013": ["a2000000-0000-0000-0000-000000000013"], + "a2000000-0000-0000-0000-000000000014": ["a2000000-0000-0000-0000-000000000014"] }, "positions": { - "b5613b69-ccc8-40a6-a7c6-8c9b9b909abf": {"col": 1, "row": 1, "height": 4, "width": "Infinity"}, - "01f7bdc5-14f0-4221-9bbf-ba3bea6b308c": {"col": 1, "row": 5, "height": 4, "width": 6}, - "1295ec49-8bb6-42d2-87ba-adad9ed43e73": {"col": 7, "row": 5, "height": 4, "width": 6}, - "d9551aab-be85-43b6-a205-02a8a6434401": {"col": 1, "row": 9, "height": 4, "width": 6}, - "8694431d-0f1a-4162-910e-de6d3fbe0b3e": {"col": 7, "row": 9, "height": 4, "width": 6} + "a2000000-0000-0000-0000-000000000011": {"col": 1, "row": 1, "height": 4, "width": "Infinity"}, + "a2000000-0000-0000-0000-000000000012": {"col": 1, "row": 5, "height": 4, "width": 6}, + "a2000000-0000-0000-0000-000000000013": {"col": 7, "row": 5, "height": 4, "width": 6}, + "a2000000-0000-0000-0000-000000000014": {"col": 1, "row": 9, "height": 4, "width": "Infinity"} }, "formatting": {"highlighting": []}, "display_mode_settings": {"positions": {}} diff --git a/dashboards/view_overview.json b/dashboards/view_overview.json new file mode 100644 index 0000000..e9821e1 --- /dev/null +++ b/dashboards/view_overview.json @@ -0,0 +1,139 @@ +{ + "entity": { + "type": "DASHBOARD", + "title": "Overview & Alerts", + "summary": "Landing dashboard - what's firing right now and overall message volume.", + "description": "Start here. Recent alerts, message volume by source (to catch floods), and a priority/critical breakdown for the last 24 hours.", + "search_id": "__SEARCH_ID__", + "properties": [], + "requires": {}, + "state": { + "a1000000-0000-0000-0000-000000000001": { + "selected_fields": null, + "static_message_list_id": null, + "titles": { + "tab": {"title": "Overview"}, + "widget": { + "a1000000-0000-0000-0000-000000000011": "Recent Alerts (24h)", + "a1000000-0000-0000-0000-000000000012": "Message Volume by Source (last hour)", + "a1000000-0000-0000-0000-000000000013": "Events by Priority (24h)", + "a1000000-0000-0000-0000-000000000014": "Critical Events by Type (24h)" + } + }, + "widgets": [ + { + "id": "a1000000-0000-0000-0000-000000000011", + "type": "messages", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 86400}, + "query": null, + "streams": ["000000000000000000000002"], + "stream_categories": [], + "config": { + "fields": ["timestamp", "priority", "message"], + "show_message_row": true, + "decorators": [], + "sort": [] + }, + "description": null, + "context": null + }, + { + "id": "a1000000-0000-0000-0000-000000000012", + "type": "aggregation", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 3600}, + "query": null, + "streams": [], + "stream_categories": [], + "config": { + "row_pivots": [{"fields": ["timestamp"], "type": "time", "config": {"interval": {"type": "auto", "scaling": 1.0}}}], + "units": {}, + "column_pivots": [{"fields": ["gl2_remote_ip"], "type": "values", "config": {"limit": 10}}], + "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], + "sort": [], + "visualization": "line", + "visualization_config": {"interpolation": "linear", "axis_type": "linear"}, + "formatting_settings": null, + "rollup": false, + "event_annotation": false, + "row_limit": null, + "column_limit": 10 + }, + "description": null, + "context": null + }, + { + "id": "a1000000-0000-0000-0000-000000000013", + "type": "aggregation", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 86400}, + "query": null, + "streams": ["000000000000000000000002"], + "stream_categories": [], + "config": { + "row_pivots": [{"fields": ["priority"], "type": "values", "config": {"limit": 10}}], + "units": {}, + "column_pivots": [], + "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "visualization": "pie", + "visualization_config": null, + "formatting_settings": null, + "rollup": true, + "event_annotation": false, + "row_limit": 10, + "column_limit": null + }, + "description": null, + "context": null + }, + { + "id": "a1000000-0000-0000-0000-000000000014", + "type": "aggregation", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 86400}, + "query": "severity_tag:critical", + "streams": [], + "stream_categories": [], + "config": { + "row_pivots": [{"fields": ["event_type"], "type": "values", "config": {"limit": 15}}], + "units": {}, + "column_pivots": [], + "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "visualization": "table", + "visualization_config": {"pinned_columns": [], "show_row_numbers": true}, + "formatting_settings": null, + "rollup": true, + "event_annotation": false, + "row_limit": 15, + "column_limit": null + }, + "description": null, + "context": null + } + ], + "widget_mapping": { + "a1000000-0000-0000-0000-000000000011": ["a1000000-0000-0000-0000-000000000011"], + "a1000000-0000-0000-0000-000000000012": ["a1000000-0000-0000-0000-000000000012"], + "a1000000-0000-0000-0000-000000000013": ["a1000000-0000-0000-0000-000000000013"], + "a1000000-0000-0000-0000-000000000014": ["a1000000-0000-0000-0000-000000000014"] + }, + "positions": { + "a1000000-0000-0000-0000-000000000011": {"col": 1, "row": 1, "height": 5, "width": "Infinity"}, + "a1000000-0000-0000-0000-000000000012": {"col": 1, "row": 6, "height": 4, "width": "Infinity"}, + "a1000000-0000-0000-0000-000000000013": {"col": 1, "row": 10, "height": 4, "width": 6}, + "a1000000-0000-0000-0000-000000000014": {"col": 7, "row": 10, "height": 4, "width": 6} + }, + "formatting": {"highlighting": []}, + "display_mode_settings": {"positions": {}} + } + } + }, + "share_request": {"selected_grantee_capabilities": {}} +} diff --git a/dashboards/view_servers.json b/dashboards/view_servers.json new file mode 100644 index 0000000..4a92216 --- /dev/null +++ b/dashboards/view_servers.json @@ -0,0 +1,147 @@ +{ + "entity": { + "type": "DASHBOARD", + "title": "Servers & Sessions", + "summary": "accel-ppp/RADIUS/conntrack - event types, top servers by volume, RADIUS accounting status breakdown.", + "description": "Scoped to the Servers stream only. Default range is 7 days. To trace one subscriber's full session, search accelppp_interface, calling_station_id, or radius_session_id on the Search page.", + "search_id": "__SEARCH_ID__", + "properties": [], + "requires": {}, + "state": { + "a3000000-0000-0000-0000-000000000001": { + "selected_fields": null, + "static_message_list_id": null, + "titles": { + "tab": {"title": "Servers & Sessions"}, + "widget": { + "a3000000-0000-0000-0000-000000000011": "Messages Over Time by Event Type", + "a3000000-0000-0000-0000-000000000012": "Event Types", + "a3000000-0000-0000-0000-000000000013": "Top Servers (by volume)", + "a3000000-0000-0000-0000-000000000014": "RADIUS Accounting Status Breakdown" + } + }, + "widgets": [ + { + "id": "a3000000-0000-0000-0000-000000000011", + "type": "aggregation", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 604800}, + "query": null, + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "config": { + "row_pivots": [{"fields": ["timestamp"], "type": "time", "config": {"interval": {"type": "auto", "scaling": 1.0}}}], + "units": {}, + "column_pivots": [{"fields": ["event_type"], "type": "values", "config": {"limit": 10}}], + "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], + "sort": [], + "visualization": "bar", + "visualization_config": {"barmode": "stack", "axis_type": "linear", "axis_config": null}, + "formatting_settings": null, + "rollup": false, + "event_annotation": false, + "row_limit": null, + "column_limit": 10 + }, + "description": null, + "context": null + }, + { + "id": "a3000000-0000-0000-0000-000000000012", + "type": "aggregation", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 604800}, + "query": null, + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "config": { + "row_pivots": [{"fields": ["event_type"], "type": "values", "config": {"limit": 15}}], + "units": {}, + "column_pivots": [], + "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "visualization": "table", + "visualization_config": {"pinned_columns": [], "show_row_numbers": true}, + "formatting_settings": null, + "rollup": true, + "event_annotation": false, + "row_limit": 15, + "column_limit": null + }, + "description": null, + "context": null + }, + { + "id": "a3000000-0000-0000-0000-000000000013", + "type": "aggregation", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 604800}, + "query": null, + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "config": { + "row_pivots": [{"fields": ["source"], "type": "values", "config": {"limit": 15}}], + "units": {}, + "column_pivots": [], + "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "visualization": "table", + "visualization_config": {"pinned_columns": [], "show_row_numbers": true}, + "formatting_settings": null, + "rollup": true, + "event_annotation": false, + "row_limit": 15, + "column_limit": null + }, + "description": null, + "context": null + }, + { + "id": "a3000000-0000-0000-0000-000000000014", + "type": "aggregation", + "filter": null, + "filters": [], + "timerange": {"type": "relative", "range": 604800}, + "query": "event_type:radius_accounting", + "streams": ["__SERVERS_STREAM_ID__"], + "stream_categories": [], + "config": { + "row_pivots": [{"fields": ["acct_status_type"], "type": "values", "config": {"limit": 10}}], + "units": {}, + "column_pivots": [], + "series": [{"config": {"name": "Message count", "thresholds": []}, "function": "count()"}], + "sort": [{"type": "series", "field": "count()", "direction": "Descending"}], + "visualization": "pie", + "visualization_config": null, + "formatting_settings": null, + "rollup": true, + "event_annotation": false, + "row_limit": 10, + "column_limit": null + }, + "description": null, + "context": null + } + ], + "widget_mapping": { + "a3000000-0000-0000-0000-000000000011": ["a3000000-0000-0000-0000-000000000011"], + "a3000000-0000-0000-0000-000000000012": ["a3000000-0000-0000-0000-000000000012"], + "a3000000-0000-0000-0000-000000000013": ["a3000000-0000-0000-0000-000000000013"], + "a3000000-0000-0000-0000-000000000014": ["a3000000-0000-0000-0000-000000000014"] + }, + "positions": { + "a3000000-0000-0000-0000-000000000011": {"col": 1, "row": 1, "height": 4, "width": "Infinity"}, + "a3000000-0000-0000-0000-000000000012": {"col": 1, "row": 5, "height": 4, "width": 6}, + "a3000000-0000-0000-0000-000000000013": {"col": 7, "row": 5, "height": 4, "width": 6}, + "a3000000-0000-0000-0000-000000000014": {"col": 1, "row": 9, "height": 4, "width": 6} + }, + "formatting": {"highlighting": []}, + "display_mode_settings": {"positions": {}} + } + } + }, + "share_request": {"selected_grantee_capabilities": {}} +} diff --git a/install-graylog.sh b/install-graylog.sh index 6c24765..1a2ad56 100644 --- a/install-graylog.sh +++ b/install-graylog.sh @@ -381,28 +381,32 @@ step_firewall() { } step_dashboard() { - if [ ! -f "$SCRIPT_DIR/dashboards/search.json" ]; then - skip "Dashboard (no dashboards/search.json in this checkout)" - return - fi - log "Creating dashboard (idempotent)..." - local title id - title="$(python3 -c "import json;print(json.load(open('$SCRIPT_DIR/dashboards/view.json'))['entity']['title'])")" - id="$(gcurl GET /views | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((v['id'] for v in d['views'] if v['title']=='$title'),''))")" - if [ -n "$id" ]; then - skip "dashboard '$title' already exists ($id)" - return - fi + # Three focused dashboards instead of one combined view - each targets a + # different reader (landing/alerts overview, network-equipment-only, + # servers-only) so opening Graylog goes straight to what's relevant + # instead of one big mixed-stream page. + local existing_titles + existing_titles="$(gcurl GET /views | python3 -c "import json,sys;print('\n'.join(v['title'] for v in json.load(sys.stdin)['views']))")" - local search_id - search_id="$(gcurl POST /views/search "$(cat "$SCRIPT_DIR/dashboards/search.json")" | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")" - [ -n "$search_id" ] || die "dashboard search creation failed" + local search_file view_file title id search_id body + for search_file in "$SCRIPT_DIR"/dashboards/search_*.json; do + view_file="${search_file/search_/view_}" + [ -f "$view_file" ] || die "missing $view_file for $search_file" + title="$(python3 -c "import json;print(json.load(open('$view_file'))['entity']['title'])")" + if echo "$existing_titles" | grep -qx "$title"; then + skip "dashboard '$title' already exists" + continue + fi - local body - body="$(sed "s/__SEARCH_ID__/$search_id/" "$SCRIPT_DIR/dashboards/view.json")" - id="$(gcurl POST /views "$body" | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")" - [ -n "$id" ] || die "dashboard '$title' failed to create" - ok "created dashboard '$title' ($id)" + body="$(sed -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/g" -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/g" "$search_file")" + search_id="$(gcurl POST /views/search "$body" | python3 -c "import json,sys;print(json.load(sys.stdin)['id'])")" + [ -n "$search_id" ] || die "search creation failed for dashboard '$title'" + + body="$(sed -e "s/__NETWORK_STREAM_ID__/$NETWORK_STREAM_ID/g" -e "s/__SERVERS_STREAM_ID__/$SERVERS_STREAM_ID/g" -e "s/__SEARCH_ID__/$search_id/" "$view_file")" + id="$(gcurl POST /views "$body" | python3 -c "import json,sys;print(json.load(sys.stdin).get('id',''))")" + [ -n "$id" ] || die "dashboard '$title' failed to create" + ok "created dashboard '$title' ($id)" + done } step_alerts() { diff --git a/pipelines/pipeline2_servers.json b/pipelines/pipeline2_servers.json index 818b32d..b524cb6 100644 --- a/pipelines/pipeline2_servers.json +++ b/pipelines/pipeline2_servers.json @@ -1,5 +1,5 @@ { "title": "Servers Parsing", "description": "Parses accel-ppp/RADIUS/conntrack syslog", - "source": "pipeline \"Servers Parsing\"\nstage 0 match either\n rule \"accelppp_router_address_error\";\n rule \"accelppp_radius_accounting\";\n rule \"accelppp_radius_server_down\";\n rule \"accelppp_no_radius_servers\";\n rule \"accelppp_auth_failed\";\n rule \"conntrack_table_full\";\n rule \"freeradius_login_ok\";\n rule \"freeradius_login_incorrect\";\n rule \"generic_critical_severity\";\nend" + "source": "pipeline \"Servers Parsing\"\nstage 0 match either\n rule \"accelppp_router_address_error\";\n rule \"accelppp_radius_accounting\";\n rule \"accelppp_radius_access_request\";\n rule \"accelppp_radius_server_down\";\n rule \"accelppp_no_radius_servers\";\n rule \"accelppp_auth_failed\";\n rule \"conntrack_table_full\";\n rule \"freeradius_login_ok\";\n rule \"freeradius_login_incorrect\";\n rule \"generic_critical_severity\";\n rule \"accelppp_interface_tag\";\nend" } diff --git a/rules/rule11_accelppp_radius_accounting.json b/rules/rule11_accelppp_radius_accounting.json index bbc1751..465b51c 100644 --- a/rules/rule11_accelppp_radius_accounting.json +++ b/rules/rule11_accelppp_radius_accounting.json @@ -1,5 +1,5 @@ { "title": "accelppp_radius_accounting", - "description": "accel-ppp: RADIUS Accounting-Request (session start/stop/alive/interim)", - "source": "rule \"accelppp_radius_accounting\"\nwhen\n contains(to_string($message.message), \"Accounting-Request\")\nthen\n set_field(\"vendor\", \"accel-ppp\");\n set_field(\"event_type\", \"radius_accounting\");\n let m = regex(\"(\\\\S+): send \\\\[RADIUS\\\\(\\\\d+\\\\) Accounting-Request.*?.*?.*?\", to_string($message.message), [\"interface\",\"nas_identifier\",\"acct_status_type\",\"framed_ip\"]);\n set_field(\"accelppp_interface\", m[\"interface\"]);\n set_field(\"nas_identifier\", m[\"nas_identifier\"]);\n set_field(\"acct_status_type\", m[\"acct_status_type\"]);\n set_field(\"framed_ip\", m[\"framed_ip\"]);\nend" + "description": "accel-ppp: RADIUS Accounting-Request (session start/stop/alive/interim) - also extracts session-correlation fields (radius_session_id, calling_station_id, radius_username) so the full session lifecycle can be searched by one key", + "source": "rule \"accelppp_radius_accounting\"\nwhen\n contains(to_string($message.message), \"Accounting-Request\")\nthen\n set_field(\"vendor\", \"accel-ppp\");\n set_field(\"event_type\", \"radius_accounting\");\n let m = regex(\"(\\\\S+): send \\\\[RADIUS\\\\(\\\\d+\\\\) Accounting-Request.*?.*?.*?.*?.*?.*?\", to_string($message.message), [\"interface\",\"radius_username\",\"nas_identifier\",\"calling_station_id\",\"acct_status_type\",\"radius_session_id\",\"framed_ip\"]);\n set_field(\"accelppp_interface\", m[\"interface\"]);\n set_field(\"radius_username\", m[\"radius_username\"]);\n set_field(\"nas_identifier\", m[\"nas_identifier\"]);\n set_field(\"calling_station_id\", m[\"calling_station_id\"]);\n set_field(\"acct_status_type\", m[\"acct_status_type\"]);\n set_field(\"radius_session_id\", m[\"radius_session_id\"]);\n set_field(\"framed_ip\", m[\"framed_ip\"]);\nend" } diff --git a/rules/rule20_accelppp_radius_access_request.json b/rules/rule20_accelppp_radius_access_request.json new file mode 100644 index 0000000..697d129 --- /dev/null +++ b/rules/rule20_accelppp_radius_access_request.json @@ -0,0 +1,5 @@ +{ + "title": "accelppp_radius_access_request", + "description": "accel-ppp: RADIUS Access-Request (subscriber authentication attempt - the session-start event, before Accounting-Request Start) - extracts the same session-correlation fields as accelppp_radius_accounting so a session can be traced from initial auth through to termination", + "source": "rule \"accelppp_radius_access_request\"\nwhen\n contains(to_string($message.message), \"Access-Request\")\nthen\n set_field(\"vendor\", \"accel-ppp\");\n set_field(\"event_type\", \"radius_access_request\");\n let m = regex(\"(\\\\S+): send \\\\[RADIUS\\\\(\\\\d+\\\\) Access-Request.*?.*?.*?\", to_string($message.message), [\"interface\",\"radius_username\",\"nas_identifier\",\"calling_station_id\"]);\n set_field(\"accelppp_interface\", m[\"interface\"]);\n set_field(\"radius_username\", m[\"radius_username\"]);\n set_field(\"nas_identifier\", m[\"nas_identifier\"]);\n set_field(\"calling_station_id\", m[\"calling_station_id\"]);\nend" +} diff --git a/rules/rule21_accelppp_interface_tag.json b/rules/rule21_accelppp_interface_tag.json new file mode 100644 index 0000000..01f40ae --- /dev/null +++ b/rules/rule21_accelppp_interface_tag.json @@ -0,0 +1,5 @@ +{ + "title": "accelppp_interface_tag", + "description": "accel-ppp: fallback tag for any log line mentioning a vlanNNNN.NNN subscriber interface that no more specific rule already classified (DHCP discover/offer/ack, ipoe session create/start/finish/terminate, etc.) - lets the full session lifecycle be searched by accelppp_interface alone, regardless of message type", + "source": "rule \"accelppp_interface_tag\"\nwhen\n !has_field(\"accelppp_interface\") && contains(to_string($message.message), \"vlan\")\nthen\n let m = regex(\"(vlan\\\\d+\\\\.\\\\d+)\", to_string($message.message), [\"iface\"]);\n set_field(\"accelppp_interface\", m[\"iface\"]);\n set_field(\"vendor\", \"accel-ppp\");\nend" +}