diff --git a/alerts/alert7_accelppp_repeated_error_warn.json b/alerts/alert7_accelppp_repeated_error_warn.json index dd6e181..ff7b17e 100644 --- a/alerts/alert7_accelppp_repeated_error_warn.json +++ b/alerts/alert7_accelppp_repeated_error_warn.json @@ -1,11 +1,11 @@ { "title": "WARNING: repeated accel-ppp error/warning from one server", - "description": "The same category of accel-ppp error/warning is repeating from one server - a single occurrence can be transient (a RADIUS retry, a one-off MAC change), but sustained repetition usually means a stuck session or an ongoing condition. Confirmed live on 2026-07-23 against a real 1GB accel-ppp log: 'can't determine router address' repeated 2,746 times over ~4 hours for two specific subscriber interfaces before self-resolving, with no alert firing at the time since this alert didn't exist yet. Threshold (>5 in 5 minutes) is set low enough to have caught that incident within its first cycle, while still tolerating an occasional single warning.", + "description": "The same category of accel-ppp error/warning is repeating from one server - a single occurrence can be transient (a RADIUS retry, a one-off MAC change), but sustained repetition usually means a stuck session or an ongoing condition. Confirmed live on 2026-07-23 against a real 1GB accel-ppp log: 'can't determine router address' repeated 2,746 times over ~4 hours for two specific subscriber interfaces before self-resolving, with no alert firing at the time since this alert didn't exist yet. Threshold (>5 in 5 minutes) is set low enough to have caught that incident within its first cycle, while still tolerating an occasional single warning. Query requires event_type to exist (not just vendor:accel-ppp) - confirmed live that vendor alone also matches routine traffic tagged only by the generic accelppp_interface_tag fallback (which never sets event_type), producing false '(Empty Value)' group_by matches otherwise.", "priority": 2, "alert": true, "config": { "type": "aggregation-v1", - "query": "vendor:accel-ppp", + "query": "vendor:accel-ppp AND event_type:*", "streams": ["__SERVERS_STREAM_ID__"], "group_by": ["gl2_remote_ip", "event_type"], "series": [{"type": "count", "id": "count-", "field": null}], diff --git a/install-graylog.sh b/install-graylog.sh index 9305e53..a094d85 100644 --- a/install-graylog.sh +++ b/install-graylog.sh @@ -315,17 +315,32 @@ step_inputs() { step_pipeline_rules() { log "Importing pipeline rules from $SCRIPT_DIR/rules/*.json (idempotent)..." - local existing_titles - existing_titles="$(gcurl GET /system/pipelines/rule | python3 -c "import json,sys;print('\n'.join(r['title'] for r in json.load(sys.stdin)))")" + # Compares by 'source' (not just title) so a rule whose regex/logic + # changed under the same title gets PUT-updated instead of silently + # skipped - this bit us live this session (rule11's session-correlation + # regex, rule18's severity fix) before this check existed. + local existing + existing="$(gcurl GET /system/pipelines/rule)" - local f title + local f title id source current_source result for f in "$SCRIPT_DIR"/rules/*.json; do title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")" - if echo "$existing_titles" | grep -qx "$title"; then - skip "rule '$title' already exists" + source="$(python3 -c "import json;print(json.load(open('$f'))['source'])")" + id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((r['id'] for r in d if r['title']=='$title'),''))")" + if [ -n "$id" ]; then + current_source="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((r['source'] for r in d if r['id']=='$id'),''))")" + if [ "$current_source" = "$source" ]; then + skip "rule '$title' already up to date" + continue + fi + result="$(gcurl PUT "/system/pipelines/rule/$id" "$(cat "$f")")" + if echo "$result" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d.get('errors') is None else 1)"; then + ok "updated rule '$title' ($id) - source changed" + else + die "rule '$title' failed to compile: $result" + fi continue fi - local result result="$(gcurl POST /system/pipelines/rule "$(cat "$f")")" if echo "$result" | python3 -c "import json,sys;d=json.load(sys.stdin);sys.exit(0 if d.get('errors') is None else 1)"; then ok "created rule '$title'" @@ -337,15 +352,29 @@ step_pipeline_rules() { step_pipelines() { log "Importing pipelines from $SCRIPT_DIR/pipelines/*.json (idempotent)..." + # Same "compare by content, PUT-update if changed" fix as + # step_pipeline_rules() - an existing pipeline's title never changes + # even when its stage list gains new rules, so a plain title-exists + # check would silently skip the update forever. Confirmed live this + # session: 5 new rules were added to rules/*.json and created fine in + # Graylog, but "Servers Parsing" kept its old stage list since this + # function only ever checked title existence before this fix. local existing existing="$(gcurl GET /system/pipelines/pipeline)" - local f title id + local f title id source current_source result for f in "$SCRIPT_DIR"/pipelines/*.json; do title="$(python3 -c "import json;print(json.load(open('$f'))['title'])")" + source="$(python3 -c "import json;print(json.load(open('$f'))['source'])")" id="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((p['id'] for p in d if p['title']=='$title'),''))")" if [ -n "$id" ]; then - skip "pipeline '$title' already exists ($id)" + current_source="$(echo "$existing" | python3 -c "import json,sys;d=json.load(sys.stdin);print(next((p['source'] for p in d if p['id']=='$id'),''))")" + if [ "$current_source" = "$source" ]; then + skip "pipeline '$title' already up to date ($id)" + else + gcurl PUT "/system/pipelines/pipeline/$id" "$(cat "$f")" >/dev/null + ok "updated pipeline '$title' ($id) - stage list changed" + fi if [ "$title" = "Network Equipment Parsing" ]; then NETWORK_PIPELINE_ID="$id"; fi if [ "$title" = "Servers Parsing" ]; then SERVERS_PIPELINE_ID="$id"; fi continue