Rework backups to use rclone, configurable entirely in the admin panel
The backup system was hard-wired to Google Drive via googleapiclient and a service account — replaced with rclone (invoked as an external binary via subprocess, credentials passed as RCLONE_CONFIG_BACKUP_* env vars, never written to a config file on disk) so the destination isn't locked to one provider. The admin panel now has a storage-type picker (Google Drive / S3- compatible / custom) with real form fields for each — no rclone.conf to paste in, matching the same non-technical-friendly spirit as the previous Google-only settings form, just generalized. bober_bbq/utils/gdrive_backup.py -> bober_bbq/utils/backup_remote.py; Setting keys generalized from gdrive_* to backup_*/backup_remote_* (clean replacement, no migration needed — confirmed via production that Google Drive backup was never enabled there). Local backup create/list/download/ delete/restore is untouched. Removes the now-unused google-api-python- client/google-auth/google-auth-httplib2 dependencies. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
699a73c8a0
commit
f53a142cf4
1 changed files with 0 additions and 253 deletions
|
|
@ -1,253 +0,0 @@
|
|||
"""Automatic off-site database backups to Google Drive via a service
|
||||
account (no interactive OAuth — runs unattended forever), with rotation:
|
||||
once more than `gdrive_retention_count` backups exist in the target
|
||||
folder, the oldest ones are deleted.
|
||||
|
||||
Entirely opt-in via "gdrive_backup_enabled". The scheduled job in
|
||||
run_web.py calls run_scheduled_backup() frequently (e.g. hourly); this
|
||||
module itself decides whether the configured interval has actually
|
||||
elapsed, so changing the interval in the admin panel takes effect
|
||||
without restarting the process.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from bober_bbq.config import BASE_DIR, config
|
||||
from bober_bbq.extensions import db
|
||||
from bober_bbq.models import Setting
|
||||
from bober_bbq.utils.telegram_notify import send_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_tz = ZoneInfo(config.TIMEZONE)
|
||||
|
||||
DRIVE_SCOPES = ["https://www.googleapis.com/auth/drive.file"]
|
||||
DRIVE_MIME = "application/octet-stream"
|
||||
|
||||
|
||||
class GDriveError(Exception):
|
||||
"""Raised for any configuration or API problem — the message is
|
||||
written straight into flash messages / owner notifications, so it's
|
||||
kept human-readable rather than a raw stack trace."""
|
||||
|
||||
|
||||
def gdrive_enabled() -> bool:
|
||||
return Setting.get_bool("gdrive_backup_enabled", False)
|
||||
|
||||
|
||||
def _credentials():
|
||||
from google.oauth2 import service_account
|
||||
|
||||
raw = Setting.get("gdrive_service_account_json", "").strip()
|
||||
if not raw:
|
||||
raise GDriveError("Не вказано ключ сервісного акаунта Google")
|
||||
try:
|
||||
info = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
raise GDriveError(f"Невалідний JSON ключа сервісного акаунта: {e}") from e
|
||||
try:
|
||||
return service_account.Credentials.from_service_account_info(info, scopes=DRIVE_SCOPES)
|
||||
except Exception as e:
|
||||
raise GDriveError(f"Не вдалося створити облікові дані з ключа: {e}") from e
|
||||
|
||||
|
||||
def _drive_service():
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
return build("drive", "v3", credentials=_credentials(), cache_discovery=False)
|
||||
|
||||
|
||||
def _folder_id() -> str:
|
||||
folder_id = Setting.get("gdrive_folder_id", "").strip()
|
||||
if not folder_id:
|
||||
raise GDriveError("Не вказано ID папки Google Drive")
|
||||
return folder_id
|
||||
|
||||
|
||||
def test_connection() -> tuple[bool, str]:
|
||||
"""Verifies the service account credentials work and the folder is
|
||||
reachable, without uploading anything."""
|
||||
try:
|
||||
service = _drive_service()
|
||||
folder_id = _folder_id()
|
||||
meta = service.files().get(fileId=folder_id, fields="id,name", supportsAllDrives=True).execute()
|
||||
return True, f"Підключення працює — папка «{meta.get('name', folder_id)}» доступна."
|
||||
except GDriveError as e:
|
||||
return False, str(e)
|
||||
except Exception as e:
|
||||
return False, f"Помилка Google Drive API: {e}"
|
||||
|
||||
|
||||
def upload_backup(path: Path) -> str:
|
||||
"""Uploads a local backup file to the configured folder, returns the
|
||||
created Drive file's id."""
|
||||
from googleapiclient.http import MediaFileUpload
|
||||
|
||||
service = _drive_service()
|
||||
media = MediaFileUpload(str(path), mimetype=DRIVE_MIME, resumable=False)
|
||||
created = (
|
||||
service.files()
|
||||
.create(body={"name": path.name, "parents": [_folder_id()]}, media_body=media, fields="id", supportsAllDrives=True)
|
||||
.execute()
|
||||
)
|
||||
return created["id"]
|
||||
|
||||
|
||||
def rotate_backups(name_contains: str) -> int:
|
||||
"""Deletes the oldest Drive backups beyond gdrive_retention_count,
|
||||
among files whose name contains `name_contains` — DB snapshots and
|
||||
uploads archives are rotated as separate groups (by their distinct
|
||||
filename patterns below) so uploading one kind never evicts the other
|
||||
out of retention. Returns how many were deleted."""
|
||||
service = _drive_service()
|
||||
retention = max(Setting.get_int("gdrive_retention_count", 14), 1)
|
||||
|
||||
files = (
|
||||
service.files()
|
||||
.list(
|
||||
q=f"'{_folder_id()}' in parents and trashed = false and name contains '{name_contains}'",
|
||||
orderBy="createdTime desc",
|
||||
fields="files(id,name,createdTime)",
|
||||
pageSize=200,
|
||||
supportsAllDrives=True,
|
||||
includeItemsFromAllDrives=True,
|
||||
)
|
||||
.execute()
|
||||
.get("files", [])
|
||||
)
|
||||
stale = files[retention:]
|
||||
for f in stale:
|
||||
service.files().delete(fileId=f["id"], supportsAllDrives=True).execute()
|
||||
return len(stale)
|
||||
|
||||
|
||||
def create_local_backup() -> Path | None:
|
||||
"""Mirrors admin/system.py's local-backup logic — kept here instead of
|
||||
imported from there so this module (utils/) never depends on admin/."""
|
||||
uri = config.SQLALCHEMY_DATABASE_URI
|
||||
if not uri.startswith("sqlite:///"):
|
||||
return None
|
||||
sqlite_path = Path(uri.replace("sqlite:///", "", 1))
|
||||
if not sqlite_path.exists():
|
||||
return None
|
||||
backup_dir = BASE_DIR / "backups"
|
||||
backup_dir.mkdir(exist_ok=True)
|
||||
stamp = datetime.now(_tz).strftime("%Y-%m-%d_%H%M%S")
|
||||
dest = backup_dir / f"bober_bbq-{stamp}-gdrive.db"
|
||||
shutil.copy2(sqlite_path, dest)
|
||||
# copy2 preserves the SOURCE db's mtime — reset it to "now" so the
|
||||
# backups list shows when this snapshot was actually taken.
|
||||
os.utime(dest, None)
|
||||
|
||||
# A backup nobody can actually restore from is worse than no backup —
|
||||
# it looks like protection while quietly being useless. Verify the copy
|
||||
# itself before it's ever uploaded, not just that the copy succeeded.
|
||||
# The connection must be closed before any unlink attempt below — on
|
||||
# Windows an open sqlite3 connection holds a file lock that makes
|
||||
# unlink() raise PermissionError even after the exception is caught.
|
||||
conn = None
|
||||
try:
|
||||
conn = sqlite3.connect(str(dest))
|
||||
result = conn.execute("PRAGMA integrity_check").fetchone()
|
||||
except sqlite3.Error as e:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
dest.unlink(missing_ok=True)
|
||||
raise GDriveError(f"Резервна копія пошкоджена (не вдалось відкрити): {e}")
|
||||
else:
|
||||
conn.close()
|
||||
if not result or result[0] != "ok":
|
||||
dest.unlink(missing_ok=True)
|
||||
raise GDriveError(f"Резервна копія пошкоджена (integrity check: {result[0] if result else '?'})")
|
||||
|
||||
return dest
|
||||
|
||||
|
||||
def create_uploads_archive() -> Path | None:
|
||||
"""Zips bober_bbq/static/uploads/ (product photos) — these were never
|
||||
part of the Drive backup before, so a DB restore alone couldn't bring
|
||||
a client's menu images back. Returns None if the folder is missing or
|
||||
has nothing but the .gitkeep placeholder in it."""
|
||||
uploads_dir = config.UPLOAD_FOLDER
|
||||
if not uploads_dir.is_dir():
|
||||
return None
|
||||
has_real_files = any(p.is_file() and p.name != ".gitkeep" for p in uploads_dir.rglob("*"))
|
||||
if not has_real_files:
|
||||
return None
|
||||
|
||||
backup_dir = BASE_DIR / "backups"
|
||||
backup_dir.mkdir(exist_ok=True)
|
||||
stamp = datetime.now(_tz).strftime("%Y-%m-%d_%H%M%S")
|
||||
archive_base = backup_dir / f"bober_bbq-uploads-{stamp}"
|
||||
archive_path = Path(shutil.make_archive(str(archive_base), "zip", root_dir=uploads_dir))
|
||||
return archive_path
|
||||
|
||||
|
||||
def backup_now() -> tuple[bool, str]:
|
||||
"""Backs up immediately, bypassing the interval check — used by the
|
||||
admin panel's "Backup зараз" button and by the scheduled job below."""
|
||||
try:
|
||||
local_path = create_local_backup()
|
||||
if local_path is None:
|
||||
return False, "Резервне копіювання в Google Drive підтримується лише для SQLite."
|
||||
upload_backup(local_path)
|
||||
deleted = rotate_backups("-gdrive.db")
|
||||
|
||||
uploads_path = create_uploads_archive()
|
||||
if uploads_path is not None:
|
||||
upload_backup(uploads_path)
|
||||
deleted += rotate_backups("bober_bbq-uploads-")
|
||||
|
||||
Setting.set("gdrive_last_backup_at", datetime.utcnow().isoformat())
|
||||
Setting.set("gdrive_last_backup_status", "ok")
|
||||
db.session.commit()
|
||||
message = f"Резервну копію завантажено в Google Drive: {local_path.name}"
|
||||
if uploads_path is not None:
|
||||
message += f" + {uploads_path.name}"
|
||||
if deleted:
|
||||
message += f" (видалено {deleted} застарілих копій)"
|
||||
logger.info(message)
|
||||
return True, message
|
||||
except GDriveError as e:
|
||||
Setting.set("gdrive_last_backup_status", f"error: {e}")
|
||||
db.session.commit()
|
||||
logger.warning("Google Drive backup failed: %s", e)
|
||||
return False, str(e)
|
||||
except Exception as e:
|
||||
Setting.set("gdrive_last_backup_status", f"error: {e}")
|
||||
db.session.commit()
|
||||
logger.exception("Google Drive backup failed")
|
||||
return False, f"Помилка Google Drive API: {e}"
|
||||
|
||||
|
||||
def run_scheduled_backup() -> None:
|
||||
"""Called periodically by APScheduler — only actually backs up once
|
||||
the configured interval has elapsed since the last successful run."""
|
||||
if not gdrive_enabled():
|
||||
return
|
||||
|
||||
interval_hours = Setting.get_int("gdrive_backup_interval_hours", 24)
|
||||
last_raw = Setting.get("gdrive_last_backup_at", "")
|
||||
if last_raw:
|
||||
try:
|
||||
last = datetime.fromisoformat(last_raw)
|
||||
if datetime.utcnow() - last < timedelta(hours=interval_hours):
|
||||
return
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
ok, message = backup_now()
|
||||
if not ok:
|
||||
for owner_id in config.OWNER_IDS:
|
||||
send_message(
|
||||
owner_id,
|
||||
f"⚠️ <b>Не вдалося зробити резервну копію в Google Drive</b>\n\n{message}\n\n"
|
||||
"Перевірте підключення: Адмінка → Резервні копії.",
|
||||
parse_mode="HTML",
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue