Add owner-alerting on unhandled errors, backup integrity checks, and a client-provisioning script

Reliability gaps found while investigating today's suppliers 500 (which I only learned about because the user reported it — nothing in the app itself said anything): - New bober_bbq/utils/error_alerts.py: notifies OWNER_IDS via the existing Telegram send_message pattern (already used by gdrive_backup/ reconciliation/checkbox_prro) on ANY unhandled request exception, via Flask's got_request_exception signal — purely observational, doesn't change the actual error response. Same helper now also wraps every APScheduler job in run_web.py, closing the one job (reconcile_payments) that had no failure handling at all and could die silently. A 15-minute per-(exception type, source) cooldown keeps a repeating failure from spamming the chat. - gdrive_backup.py: verifies each local DB backup with PRAGMA integrity_check before it's ever uploaded (a corrupt copy now fails loudly instead of silently becoming an unusable "backup"), and now also archives+uploads static/uploads/ (product photos) alongside the DB — previously never backed up at all. Also, per the earlier per-client template-readiness pass: provision_client.sh automates docs/DEPLOYMENT.md's clone/.env/nginx/TLS/systemd steps for a NEW client deployment (confirmation prompt before every irreversible step; refuses to run against an existing install-dir). Not executed anywhere this session — no target VPS yet, verified via `bash -n` and a step-by-step review against the runbook it automates. All of this is additive and was smoke-tested to confirm zero behavior change for the live instance: same error responses, same backup content, same scheduler behavior when nothing is actually broken.
2026-08-25 18:11:15 +03:00 · 2026-08-25 18:11:15 +03:00 · c1e3e83d55
commit c1e3e83d55
parent 922d9c5172
6 changed files with 366 additions and 16 deletions
--- a/bober_bbq/app.py
+++ b/bober_bbq/app.py
@ -3,7 +3,7 @@ import logging
 import os
 from pathlib import Path

-from flask import Flask, Response, jsonify, render_template_string, request, send_from_directory
+from flask import Flask, Response, got_request_exception, jsonify, render_template_string, request, send_from_directory
 from flask_wtf.csrf import CSRFProtect, generate_csrf
 from markupsafe import Markup
 from werkzeug.middleware.proxy_fix import ProxyFix
@ -19,6 +19,7 @@ from bober_bbq.api.settings import settings_bp
 from bober_bbq.config import config
 from bober_bbq.extensions import db, login_manager
 from bober_bbq.models import AdminUser, Setting
+from bober_bbq.utils import error_alerts

 csrf = CSRFProtect()

@ -150,6 +151,24 @@ def create_app() -> Flask:
    app.register_blueprint(api_bp)
    app.register_blueprint(admin_bp)

+    def _on_request_exception(sender, exception, **extra):
+        # Fires for any exception that reaches Flask's default error
+        # handling — i.e. a genuine unhandled bug, not a routine 404/403/
+        # login-redirect (those are HTTPException and never get here).
+        # Purely observational: doesn't touch the response, so the actual
+        # error page shown to the user is unchanged.
+        from werkzeug.exceptions import HTTPException
+
+        if isinstance(exception, HTTPException):
+            return
+        error_alerts.notify(f"{request.method} {request.path}", exception)
+
+    # weak=False — this is a local closure with no other reference kept
+    # anywhere; blinker's default weak reference would let it get garbage
+    # collected right after create_app() returns, silently disabling the
+    # connection (confirmed via a smoke test before adding this).
+    got_request_exception.connect(_on_request_exception, app, weak=False)
+
    @app.after_request
    def prevent_stale_shell_caching(response):
        path = request.path
--- a/bober_bbq/utils/error_alerts.py
+++ b/bober_bbq/utils/error_alerts.py
@ -0,0 +1,47 @@
+"""Alerts the owner(s) in Telegram when something breaks unattended — an
+unhandled Flask request error or a crashed background job. Both call the
+same `notify()` so there's one cooldown mechanism, not two.
+
+Before this, a failure here was visible only in the server's log file —
+finding out about a live 500 meant someone noticing and saying something,
+or an admin manually grepping `/var/log/bober-bbq-web.log` over SSH.
+"""
+
+import logging
+import time
+
+from bober_bbq.config import config
+from bober_bbq.utils.telegram_notify import send_message
+
+logger = logging.getLogger(__name__)
+
+# One alert per (exception type, source) combination per window — a bug
+# that fires on every request/job tick shouldn't flood the chat.
+_COOLDOWN_SECONDS = 15 * 60
+_last_sent: dict[tuple[str, str], float] = {}
+
+
+def notify(source: str, exc: BaseException) -> None:
+    """`source` is a short label identifying where this broke — a request
+    path (e.g. "POST /admin/suppliers/import") or a scheduled job name
+    (e.g. "reconcile_payments")."""
+    key = (type(exc).__name__, source)
+    now = time.monotonic()
+    last = _last_sent.get(key)
+    if last is not None and now - last < _COOLDOWN_SECONDS:
+        return
+    _last_sent[key] = now
+
+    text = (
+        f"🔥 <b>Помилка в застосунку</b>\n\n"
+        f"<b>Де:</b> {source}\n"
+        f"<b>Тип:</b> {type(exc).__name__}\n"
+        f"<b>Повідомлення:</b> {str(exc)[:300]}\n\n"
+        f"Деталі — в логах сервера (Адмінка → Система)."
+    )
+    for owner_id in config.OWNER_IDS:
+        try:
+            send_message(owner_id, text, parse_mode="HTML")
+        except Exception:
+            # Never let the act of reporting a failure cause another one.
+            logger.exception("Failed to send error alert to owner %s", owner_id)
--- a/bober_bbq/utils/gdrive_backup.py
+++ b/bober_bbq/utils/gdrive_backup.py
@ -14,6 +14,7 @@ import json
 import logging
 import os
 import shutil
+import sqlite3
 from datetime import datetime, timedelta
 from pathlib import Path
 from zoneinfo import ZoneInfo
@ -98,16 +99,19 @@ def upload_backup(path: Path) -> str:
    return created["id"]


-def rotate_backups() -> int:
-    """Deletes the oldest Drive backups beyond gdrive_retention_count.
-    Returns how many were deleted."""
+def rotate_backups(name_contains: str) -> int:
+    """Deletes the oldest Drive backups beyond gdrive_retention_count,
+    among files whose name contains `name_contains` — DB snapshots and
+    uploads archives are rotated as separate groups (by their distinct
+    filename patterns below) so uploading one kind never evicts the other
+    out of retention. Returns how many were deleted."""
    service = _drive_service()
    retention = max(Setting.get_int("gdrive_retention_count", 14), 1)

    files = (
        service.files()
        .list(
-            q=f"'{_folder_id()}' in parents and trashed = false",
+            q=f"'{_folder_id()}' in parents and trashed = false and name contains '{name_contains}'",
            orderBy="createdTime desc",
            fields="files(id,name,createdTime)",
            pageSize=200,
@ -140,23 +144,72 @@ def create_local_backup() -> Path | None:
    # copy2 preserves the SOURCE db's mtime — reset it to "now" so the
    # backups list shows when this snapshot was actually taken.
    os.utime(dest, None)
+
+    # A backup nobody can actually restore from is worse than no backup —
+    # it looks like protection while quietly being useless. Verify the copy
+    # itself before it's ever uploaded, not just that the copy succeeded.
+    # The connection must be closed before any unlink attempt below — on
+    # Windows an open sqlite3 connection holds a file lock that makes
+    # unlink() raise PermissionError even after the exception is caught.
+    conn = None
+    try:
+        conn = sqlite3.connect(str(dest))
+        result = conn.execute("PRAGMA integrity_check").fetchone()
+    except sqlite3.Error as e:
+        if conn is not None:
+            conn.close()
+        dest.unlink(missing_ok=True)
+        raise GDriveError(f"Резервна копія пошкоджена (не вдалось відкрити): {e}")
+    else:
+        conn.close()
+    if not result or result[0] != "ok":
+        dest.unlink(missing_ok=True)
+        raise GDriveError(f"Резервна копія пошкоджена (integrity check: {result[0] if result else '?'})")
+
    return dest


+def create_uploads_archive() -> Path | None:
+    """Zips bober_bbq/static/uploads/ (product photos) — these were never
+    part of the Drive backup before, so a DB restore alone couldn't bring
+    a client's menu images back. Returns None if the folder is missing or
+    has nothing but the .gitkeep placeholder in it."""
+    uploads_dir = config.UPLOAD_FOLDER
+    if not uploads_dir.is_dir():
+        return None
+    has_real_files = any(p.is_file() and p.name != ".gitkeep" for p in uploads_dir.rglob("*"))
+    if not has_real_files:
+        return None
+
+    backup_dir = BASE_DIR / "backups"
+    backup_dir.mkdir(exist_ok=True)
+    stamp = datetime.now(_tz).strftime("%Y-%m-%d_%H%M%S")
+    archive_base = backup_dir / f"bober_bbq-uploads-{stamp}"
+    archive_path = Path(shutil.make_archive(str(archive_base), "zip", root_dir=uploads_dir))
+    return archive_path
+
+
 def backup_now() -> tuple[bool, str]:
    """Backs up immediately, bypassing the interval check — used by the
    admin panel's "Backup зараз" button and by the scheduled job below."""
-    local_path = create_local_backup()
-    if local_path is None:
-        return False, "Резервне копіювання в Google Drive підтримується лише для SQLite."
-
    try:
+        local_path = create_local_backup()
+        if local_path is None:
+            return False, "Резервне копіювання в Google Drive підтримується лише для SQLite."
        upload_backup(local_path)
-        deleted = rotate_backups()
+        deleted = rotate_backups("-gdrive.db")
+
+        uploads_path = create_uploads_archive()
+        if uploads_path is not None:
+            upload_backup(uploads_path)
+            deleted += rotate_backups("bober_bbq-uploads-")
+
        Setting.set("gdrive_last_backup_at", datetime.utcnow().isoformat())
        Setting.set("gdrive_last_backup_status", "ok")
        db.session.commit()
        message = f"Резервну копію завантажено в Google Drive: {local_path.name}"
+        if uploads_path is not None:
+            message += f" + {uploads_path.name}"
        if deleted:
            message += f" (видалено {deleted} застарілих копій)"
        logger.info(message)
--- a/docs/DEPLOYMENT.md
+++ b/docs/DEPLOYMENT.md
@ -11,6 +11,13 @@
 Далі `$INSTALL_DIR` — шлях, куди клонується репозиторій (напр.
 `/opt/<client-slug>-bot`); підставляйте свій на кожному кроці.

+**Кроки 2-5 (клон, `.env`, nginx, TLS, systemd) автоматизовані —**
+`./provision_client.sh --slug vegcafe --domain vegcafe.example --bot-token
+··· --cafe-name "Веге Кафе"` (`--help` для повного списку прапорців).
+Питає підтвердження перед кожним незворотним кроком і ніколи не чіпає
+директорію, яка вже існує. Розділи нижче лишаються довідкою — що саме
+робить скрипт і що робити, якщо якийсь крок треба виконати вручну.
+
 ## 1. Підготовка сервера

 ```bash
--- a/provision_client.sh
+++ b/provision_client.sh
@ -0,0 +1,212 @@
+#!/bin/bash
+# Provisions a NEW, separate deployment of this codebase for another client
+# on a fresh VPS — clone, .env, webapp build, nginx vhost, TLS, systemd
+# units. Automates docs/DEPLOYMENT.md steps 2-5; run this instead of doing
+# that runbook by hand.
+#
+# Prerequisite (docs/DEPLOYMENT.md step 1, one-time OS setup, not run by
+# this script): python3.11, nginx, certbot, nodejs/npm already installed
+# on this server.
+#
+# Never touches an existing deployment: refuses to run if --install-dir
+# already exists, and every irreversible/system-level step (nginx reload,
+# certbot, systemctl enable) asks for confirmation first.
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: ./provision_client.sh --slug SLUG --domain DOMAIN --bot-token TOKEN --cafe-name "Name" [--install-dir DIR] [--repo-url URL]
+
+  --slug        short identifier, lowercase letters/digits/hyphens only
+                (used for the install dir and systemd unit names)
+  --domain      public domain this instance will be served from
+  --bot-token   Telegram bot token from @BotFather
+  --cafe-name   display name for this client (webapp title, PWA manifest)
+  --install-dir defaults to /opt/<slug>-bot
+  --repo-url    defaults to this project's own git remote
+EOF
+}
+
+SLUG=""
+DOMAIN=""
+BOT_TOKEN=""
+CAFE_NAME=""
+INSTALL_DIR=""
+REPO_URL="https://git.zotac.keenetic.link/zotac/bober-bbq-bot.git"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --slug) SLUG="$2"; shift 2 ;;
+    --domain) DOMAIN="$2"; shift 2 ;;
+    --bot-token) BOT_TOKEN="$2"; shift 2 ;;
+    --cafe-name) CAFE_NAME="$2"; shift 2 ;;
+    --install-dir) INSTALL_DIR="$2"; shift 2 ;;
+    --repo-url) REPO_URL="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "Невідомий аргумент: $1"; usage; exit 1 ;;
+  esac
+done
+
+if [[ -z "$SLUG" || -z "$DOMAIN" || -z "$BOT_TOKEN" || -z "$CAFE_NAME" ]]; then
+  echo "Не вистачає обов'язкового аргументу."
+  usage
+  exit 1
+fi
+if [[ ! "$SLUG" =~ ^[a-z0-9-]+$ ]]; then
+  echo "--slug має містити лише малі латинські літери, цифри й дефіс (напр. \"vegcafe\")."
+  exit 1
+fi
+
+INSTALL_DIR="${INSTALL_DIR:-/opt/${SLUG}-bot}"
+SERVICE_WEB="${SLUG}-web"
+SERVICE_BOT="${SLUG}-bot"
+LOG_FILE_WEB="/var/log/${SERVICE_WEB}.log"
+LOG_FILE_BOT="/var/log/${SERVICE_BOT}.log"
+
+confirm() {
+  local reply
+  read -rp "$1 [y/N] " reply
+  [[ "$reply" =~ ^[Yy]$ ]]
+}
+
+echo "=== Провижнінг нового клієнта ==="
+echo "Slug:        $SLUG"
+echo "Домен:       $DOMAIN"
+echo "Install dir: $INSTALL_DIR"
+echo "Cafe name:   $CAFE_NAME"
+echo "Юніти:       $SERVICE_WEB / $SERVICE_BOT"
+echo ""
+if ! confirm "Продовжити?"; then
+  echo "Скасовано."
+  exit 0
+fi
+
+if [[ -e "$INSTALL_DIR" ]]; then
+  echo "ПОМИЛКА: $INSTALL_DIR вже існує — оберіть інший --install-dir."
+  echo "(Це також запобігає випадковому запуску поверх існуючого деплою.)"
+  exit 1
+fi
+
+echo "==> 1/8 git clone"
+git clone "$REPO_URL" "$INSTALL_DIR"
+cd "$INSTALL_DIR"
+
+echo "==> 2/8 venv + залежності"
+python3.11 -m venv .venv
+.venv/bin/pip install -q -r requirements.txt
+
+echo "==> 3/8 .env"
+SECRET_KEY=$(openssl rand -hex 32)
+ADMIN_PASSWORD=$(openssl rand -hex 12)
+cp .env.example .env
+sed -i "s|^BOT_TOKEN=.*|BOT_TOKEN=${BOT_TOKEN}|" .env
+sed -i "s|^WEBAPP_URL=.*|WEBAPP_URL=https://${DOMAIN}/webapp/|" .env
+sed -i "s|^BACKEND_PUBLIC_URL=.*|BACKEND_PUBLIC_URL=https://${DOMAIN}|" .env
+sed -i "s|^MONOBANK_WEBHOOK_URL=.*|MONOBANK_WEBHOOK_URL=https://${DOMAIN}/api/payments/monobank/webhook|" .env
+sed -i "s|^SECRET_KEY=.*|SECRET_KEY=${SECRET_KEY}|" .env
+sed -i "s|^FLASK_ADMIN_PASSWORD=.*|FLASK_ADMIN_PASSWORD=${ADMIN_PASSWORD}|" .env
+{
+  echo ""
+  echo "# --- provision_client.sh: deployment identity ---"
+  echo "SERVICE_WEB=${SERVICE_WEB}"
+  echo "SERVICE_BOT=${SERVICE_BOT}"
+  echo "LOG_FILE_WEB=${LOG_FILE_WEB}"
+  echo "LOG_FILE_BOT=${LOG_FILE_BOT}"
+} >> .env
+echo "Адмін-логін: admin / пароль: ${ADMIN_PASSWORD}  (запишіть — більше ніде не показується)"
+
+echo "==> 4/8 webapp/.env.local"
+echo "VITE_CAFE_NAME=${CAFE_NAME}" > webapp/.env.local
+
+echo "==> 5/8 webapp build"
+( cd webapp && npm install --silent && npm run build )
+
+echo "==> 6/8 nginx vhost"
+if confirm "Створити nginx vhost для ${DOMAIN} і перезавантажити nginx?"; then
+  NGINX_CONF="/etc/nginx/sites-available/${SLUG}"
+  sudo tee "$NGINX_CONF" > /dev/null <<EOF
+server {
+    listen 80;
+    server_name ${DOMAIN};
+
+    location / {
+        proxy_pass http://127.0.0.1:8000;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-Proto \$scheme;
+    }
+}
+EOF
+  sudo ln -sf "$NGINX_CONF" "/etc/nginx/sites-enabled/${SLUG}"
+  sudo nginx -t
+  sudo systemctl reload nginx
+else
+  echo "Пропущено — налаштуйте nginx вручну (docs/DEPLOYMENT.md, крок 4)."
+fi
+
+echo "==> 7/8 certbot (TLS)"
+if confirm "Запустити certbot для ${DOMAIN}? (потрібен вже налаштований DNS-запис на цей сервер)"; then
+  sudo certbot --nginx -d "$DOMAIN"
+else
+  echo "Пропущено — без TLS Telegram Mini App і вебхуки не запрацюють."
+  echo "Запустіть пізніше: sudo certbot --nginx -d ${DOMAIN}"
+fi
+
+echo "==> 8/8 systemd-юніти"
+if confirm "Створити й запустити systemd-юніти (${SERVICE_WEB}, ${SERVICE_BOT})?"; then
+  sudo tee "/etc/systemd/system/${SERVICE_WEB}.service" > /dev/null <<EOF
+[Unit]
+Description=${CAFE_NAME} backend (API + admin)
+After=network.target
+
+[Service]
+WorkingDirectory=${INSTALL_DIR}
+ExecStart=${INSTALL_DIR}/.venv/bin/python run_web.py
+Restart=on-failure
+EnvironmentFile=${INSTALL_DIR}/.env
+StandardOutput=append:${LOG_FILE_WEB}
+StandardError=append:${LOG_FILE_WEB}
+
+[Install]
+WantedBy=multi-user.target
+EOF
+  sudo tee "/etc/systemd/system/${SERVICE_BOT}.service" > /dev/null <<EOF
+[Unit]
+Description=${CAFE_NAME} Telegram bot
+After=network.target ${SERVICE_WEB}.service
+
+[Service]
+WorkingDirectory=${INSTALL_DIR}
+ExecStart=${INSTALL_DIR}/.venv/bin/python run_bot.py
+Restart=on-failure
+EnvironmentFile=${INSTALL_DIR}/.env
+StandardOutput=append:${LOG_FILE_BOT}
+StandardError=append:${LOG_FILE_BOT}
+
+[Install]
+WantedBy=multi-user.target
+EOF
+  sudo systemctl daemon-reload
+  sudo systemctl enable --now "$SERVICE_WEB" "$SERVICE_BOT"
+else
+  echo "Пропущено — створіть юніти вручну (docs/DEPLOYMENT.md, крок 5)."
+fi
+
+cat <<EOF
+
+=== Готово. Залишились ручні кроки (свідомо не автоматизуються) ===
+
+1. У @BotFather: /setmenubutton -> https://${DOMAIN}/webapp/
+2. Додати бота в чати доставки/самовивозу, вписати chat_id в Адмінку -> Налаштування.
+3. Якщо клієнт приймає картки — MONOBANK_TOKEN у .env і перевірити вебхук.
+4. Замінити 4 іконки під webapp/public/icons/icon-{32,180,192,512}.png на бренд
+   клієнта, потім ще раз: cd webapp && npm run build
+5. Адмінка (https://${DOMAIN}/admin, admin / пароль вище) -> Налаштування:
+   cafe_name, адреса, графік, кольори, лого.
+6. Адмінка -> Меню: видалити демо-меню (Шашлик/BBQ), додати асортимент клієнта.
+7. Зареєструвати https://${DOMAIN}/health в UptimeRobot (чи аналог) — вручну,
+   акаунт створює власник бізнесу, не цей скрипт.
+8. Резервні копії: Адмінка -> Резервні копії -> увімкнути Google Drive backup
+   (або окремий cron — docs/DEPLOYMENT.md, крок 7).
+
+EOF
--- a/run_web.py
+++ b/run_web.py
@ -4,6 +4,7 @@ Dev:   python bober_bbq/app.py        (Flask debug server, auto-reload)
 Prod:  python run_web.py              (waitress, no reload, multi-threaded)
 """

+import logging
 import os

 from apscheduler.schedulers.background import BackgroundScheduler
@ -15,10 +16,13 @@ from bober_bbq.extensions import db
 from bober_bbq.migrate import migrate
 from bober_bbq.models import Setting
 from bober_bbq.seed import seed
+from bober_bbq.utils import error_alerts
 from bober_bbq.utils.checkbox_prro import maybe_close_shift
 from bober_bbq.utils.gdrive_backup import run_scheduled_backup
 from bober_bbq.utils.reconciliation import check_monobank_token_health, reconcile_pending_payments

+logger = logging.getLogger(__name__)
+
 app = create_app()

 with app.app_context():
@ -27,33 +31,41 @@ with app.app_context():
    seed()


-def _run_in_app_context(fn):
+def _run_in_app_context(fn, job_name):
+    # APScheduler's default executor only logs a job's exception itself —
+    # nothing here previously caught one, so a crashing job (like
+    # reconcile_payments hitting something other than MonobankError) failed
+    # silently from the owner's perspective, visible only in server logs.
    with app.app_context():
-        fn()
+        try:
+            fn()
+        except Exception as e:
+            logger.exception("Scheduled job %s crashed", job_name)
+            error_alerts.notify(f"фонове завдання {job_name}", e)


 if __name__ == "__main__":
    scheduler = BackgroundScheduler(timezone=config.TIMEZONE)
    scheduler.add_job(
-        lambda: _run_in_app_context(reconcile_pending_payments),
+        lambda: _run_in_app_context(reconcile_pending_payments, "reconcile_payments"),
        "interval",
        minutes=10,
        id="reconcile_payments",
    )
    scheduler.add_job(
-        lambda: _run_in_app_context(check_monobank_token_health),
+        lambda: _run_in_app_context(check_monobank_token_health, "monobank_token_health"),
        "interval",
        hours=1,
        id="monobank_token_health",
    )
    scheduler.add_job(
-        lambda: _run_in_app_context(run_scheduled_backup),
+        lambda: _run_in_app_context(run_scheduled_backup, "gdrive_backup"),
        "interval",
        hours=1,
        id="gdrive_backup",
    )
    scheduler.add_job(
-        lambda: _run_in_app_context(maybe_close_shift),
+        lambda: _run_in_app_context(maybe_close_shift, "checkbox_auto_close_shift"),
        "interval",
        minutes=10,
        id="checkbox_auto_close_shift",