Fix two bugs found live: pipeline/rule update gap, and alert7's over-broad query
- step_pipeline_rules() and step_pipelines() only ever created new rules/pipelines by title, never updated existing ones whose content changed under the same title. Confirmed live this session: 5 new rules were added to rules/*.json and created fine in Graylog, but "Servers Parsing"'s stage list was never updated to actually call them, since the pipeline already existed. Both functions now compare by 'source' content and PUT-update if it differs, matching the pattern already used elsewhere (e.g. step_inputs()'s timezone self-heal). - alert7's query (vendor:accel-ppp) was too broad: it also matches routine traffic tagged only by the generic accelppp_interface_tag fallback rule, which sets vendor but never event_type - producing false "(Empty Value)" group_by matches on ordinary volume. Confirmed live. Fixed to vendor:accel-ppp AND event_type:* - severity_tag was considered instead but rejected, since accelppp_router_address_error (the rule that inspired this alert) never set it either. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
31258eee01
commit
609ac74574
2 changed files with 39 additions and 10 deletions
|
|
@ -1,11 +1,11 @@
|
|||
{
|
||||
"title": "WARNING: repeated accel-ppp error/warning from one server",
|
||||
"description": "The same category of accel-ppp error/warning is repeating from one server - a single occurrence can be transient (a RADIUS retry, a one-off MAC change), but sustained repetition usually means a stuck session or an ongoing condition. Confirmed live on 2026-07-23 against a real 1GB accel-ppp log: 'can't determine router address' repeated 2,746 times over ~4 hours for two specific subscriber interfaces before self-resolving, with no alert firing at the time since this alert didn't exist yet. Threshold (>5 in 5 minutes) is set low enough to have caught that incident within its first cycle, while still tolerating an occasional single warning.",
|
||||
"description": "The same category of accel-ppp error/warning is repeating from one server - a single occurrence can be transient (a RADIUS retry, a one-off MAC change), but sustained repetition usually means a stuck session or an ongoing condition. Confirmed live on 2026-07-23 against a real 1GB accel-ppp log: 'can't determine router address' repeated 2,746 times over ~4 hours for two specific subscriber interfaces before self-resolving, with no alert firing at the time since this alert didn't exist yet. Threshold (>5 in 5 minutes) is set low enough to have caught that incident within its first cycle, while still tolerating an occasional single warning. Query requires event_type to exist (not just vendor:accel-ppp) - confirmed live that vendor alone also matches routine traffic tagged only by the generic accelppp_interface_tag fallback (which never sets event_type), producing false '(Empty Value)' group_by matches otherwise.",
|
||||
"priority": 2,
|
||||
"alert": true,
|
||||
"config": {
|
||||
"type": "aggregation-v1",
|
||||
"query": "vendor:accel-ppp",
|
||||
"query": "vendor:accel-ppp AND event_type:*",
|
||||
"streams": ["__SERVERS_STREAM_ID__"],
|
||||
"group_by": ["gl2_remote_ip", "event_type"],
|
||||
"series": [{"type": "count", "id": "count-", "field": null}],
|
||||
|
|
|
|||
|
|
@ -315,17 +315,32 @@ step_inputs() {
|
|||
|
||||
step_pipeline_rules() {
|
||||
log "Importing pipeline rules from $SCRIPT_DIR/rules/*.json (idempotent)..."
|
||||
local existing_titles
|
||||
existing_titles="$(gcurl GET /system/pipelines/rule | python3 -c "import json,sys;print('\n'.join(r['title'] for r in json.load(sys.stdin)))")"
|
||||
# Compares by 'source' (not just title) so a rule whose regex/logic
|
||||
# changed under the same title gets PUT-updated instead of silently
|
||||
# skipped - this bit us live this session (rule11's session-correlation
|
||||
# regex, rule18's severity fix) before this check existed.
|
||||
local existing
|
||||
existing="$(gcurl GET /system/pipelines/rule)"
|
||||
|
||||
local f title
|
||||
local f title id source current_source result
|
||||
for f in "$SCRIPT_DIR"/rules/*.json; do
|
||||
title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")"
|
||||
if echo "$existing_titles" | grep -qx "$title"; then
|
||||
skip "rule '$title' already exists"
|
||||
source="$(python3 -c "import json;print(json.load(open('$f'))['source'])")"
|
||||
id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((r['id'] for r in d if r['title']=='$title'),''))")"
|
||||
if [ -n "$id" ]; then
|
||||
current_source="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((r['source'] for r in d if r['id']=='$id'),''))")"
|
||||
if [ "$current_source" = "$source" ]; then
|
||||
skip "rule '$title' already up to date"
|
||||
continue
|
||||
fi
|
||||
result="$(gcurl PUT "/system/pipelines/rule/$id" "$(cat "$f")")"
|
||||
if echo "$result" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d.get('errors') is None else 1)"; then
|
||||
ok "updated rule '$title' ($id) - source changed"
|
||||
else
|
||||
die "rule '$title' failed to compile: $result"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
local result
|
||||
result="$(gcurl POST /system/pipelines/rule "$(cat "$f")")"
|
||||
if echo "$result" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d.get('errors') is None else 1)"; then
|
||||
ok "created rule '$title'"
|
||||
|
|
@ -337,15 +352,29 @@ step_pipeline_rules() {
|
|||
|
||||
step_pipelines() {
|
||||
log "Importing pipelines from $SCRIPT_DIR/pipelines/*.json (idempotent)..."
|
||||
# Same "compare by content, PUT-update if changed" fix as
|
||||
# step_pipeline_rules() - an existing pipeline's title never changes
|
||||
# even when its stage list gains new rules, so a plain title-exists
|
||||
# check would silently skip the update forever. Confirmed live this
|
||||
# session: 5 new rules were added to rules/*.json and created fine in
|
||||
# Graylog, but "Servers Parsing" kept its old stage list since this
|
||||
# function only ever checked title existence before this fix.
|
||||
local existing
|
||||
existing="$(gcurl GET /system/pipelines/pipeline)"
|
||||
|
||||
local f title id
|
||||
local f title id source current_source result
|
||||
for f in "$SCRIPT_DIR"/pipelines/*.json; do
|
||||
title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")"
|
||||
source="$(python3 -c "import json;print(json.load(open('$f'))['source'])")"
|
||||
id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((p['id'] for p in d if p['title']=='$title'),''))")"
|
||||
if [ -n "$id" ]; then
|
||||
skip "pipeline '$title' already exists ($id)"
|
||||
current_source="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((p['source'] for p in d if p['id']=='$id'),''))")"
|
||||
if [ "$current_source" = "$source" ]; then
|
||||
skip "pipeline '$title' already up to date ($id)"
|
||||
else
|
||||
gcurl PUT "/system/pipelines/pipeline/$id" "$(cat "$f")" >/dev/null
|
||||
ok "updated pipeline '$title' ($id) - stage list changed"
|
||||
fi
|
||||
if [ "$title" = "Network Equipment Parsing" ]; then NETWORK_PIPELINE_ID="$id"; fi
|
||||
if [ "$title" = "Servers Parsing" ]; then SERVERS_PIPELINE_ID="$id"; fi
|
||||
continue
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue