From 4316ee4330a5b053c1cdbb8986bc5e206a5a0465 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Mon, 19 Oct 2020 22:17:47 +0200 Subject: [PATCH] root: implement db backups with monitored task, update docs --- docs/maintenance/backups/index.md | 25 ++++-------- helm/templates/cronjob-backup.yaml | 42 -------------------- passbook/lib/{tasks.py => tasks/__init__.py} | 0 passbook/lib/tasks/backup.py | 34 ++++++++++++++++ passbook/root/settings.py | 10 ++++- 5 files changed, 50 insertions(+), 61 deletions(-) delete mode 100644 helm/templates/cronjob-backup.yaml rename passbook/lib/{tasks.py => tasks/__init__.py} (100%) create mode 100644 passbook/lib/tasks/backup.py diff --git a/docs/maintenance/backups/index.md b/docs/maintenance/backups/index.md index 10ca567ff..1bf3ef19c 100644 --- a/docs/maintenance/backups/index.md +++ b/docs/maintenance/backups/index.md @@ -6,6 +6,10 @@ ### Backup +!!! notice + + Local backups are **enabled** by default, and will be run daily at 00:00 + Local backups can be created by running the following command in your passbook installation directory ``` @@ -14,15 +18,6 @@ docker-compose run --rm worker backup This will dump the current database into the `./backups` folder. By defaults, the last 10 Backups are kept. -To schedule these backups, use the following snippet in a crontab - -``` -0 0 * * * bash -c "cd && docker-compose run --rm worker backup" >/dev/null -``` - -!!! notice - - passbook does support automatic backups on a schedule, however this is currently not recommended, as there is no way to monitor these scheduled tasks. ### Restore @@ -42,11 +37,7 @@ After you've restored the backup, it is recommended to restart all services with ### S3 Configuration -!!! notice - - To trigger backups with S3 enabled, use the same commands as above. - -#### S3 Preparation +#### Preparation passbook expects the bucket you select to already exist. The IAM User given to passbook should have the following permissions @@ -101,11 +92,11 @@ Simply enable these options in your values.yaml file ```yaml # Enable Database Backups to S3 backup: - access_key: access-key - secret_key: secret-key + accessKey: access-key + secretKey: secret-key bucket: s3-bucket region: eu-central-1 host: s3-host ``` -Afterwards, run a `helm upgrade` to update the ConfigMap. Because passbook-scheduled backups are not recommended currently, a Kubernetes CronJob is created that runs the backup daily. +Afterwards, run a `helm upgrade` to update the ConfigMap. Backups are done automatically as above, at 00:00 every day. diff --git a/helm/templates/cronjob-backup.yaml b/helm/templates/cronjob-backup.yaml deleted file mode 100644 index 5a06a7d07..000000000 --- a/helm/templates/cronjob-backup.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{- if .Values.backup }} -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: {{ include "passbook.fullname" . }}-backup - labels: - app.kubernetes.io/name: {{ include "passbook.name" . }} - helm.sh/chart: {{ include "passbook.chart" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - schedule: "0 0 * * *" - jobTemplate: - spec: - template: - spec: - restartPolicy: Never - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.name }}:{{ .Values.image.tag }}" - args: [server] - envFrom: - - configMapRef: - name: {{ include "passbook.fullname" . }}-config - prefix: PASSBOOK_ - env: - - name: PASSBOOK_SECRET_KEY - valueFrom: - secretKeyRef: - name: "{{ include "passbook.fullname" . }}-secret-key" - key: "secret_key" - - name: PASSBOOK_REDIS__PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-redis" - key: "redis-password" - - name: PASSBOOK_POSTGRESQL__PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-postgresql" - key: "postgresql-password" -{{- end}} diff --git a/passbook/lib/tasks.py b/passbook/lib/tasks/__init__.py similarity index 100% rename from passbook/lib/tasks.py rename to passbook/lib/tasks/__init__.py diff --git a/passbook/lib/tasks/backup.py b/passbook/lib/tasks/backup.py new file mode 100644 index 000000000..94707df54 --- /dev/null +++ b/passbook/lib/tasks/backup.py @@ -0,0 +1,34 @@ +"""Database backup task""" +from datetime import datetime +from io import StringIO + +from botocore.exceptions import BotoCoreError, ClientError +from django.contrib.humanize.templatetags.humanize import naturaltime +from django.core import management +from structlog import get_logger + +from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus +from passbook.root.celery import CELERY_APP + +LOGGER = get_logger() + + +@CELERY_APP.task(bind=True, base=MonitoredTask) +def backup_database(self: MonitoredTask): # pragma: no cover + """Database backup""" + try: + start = datetime.now() + out = StringIO() + management.call_command("dbbackup", quiet=True, stdout=out) + self.set_status( + TaskResult( + TaskResultStatus.SUCCESSFUL, + [ + f"Successfully finished database backup {naturaltime(start)}", + out.getvalue(), + ], + ) + ) + LOGGER.info("Successfully backed up database.") + except (IOError, BotoCoreError, ClientError) as exc: + self.set_status(TaskResult(TaskResultStatus.ERROR).with_error(exc)) diff --git a/passbook/root/settings.py b/passbook/root/settings.py index 219f04484..9e4e46ed5 100644 --- a/passbook/root/settings.py +++ b/passbook/root/settings.py @@ -269,9 +269,14 @@ CELERY_TASK_SOFT_TIME_LIMIT = 600 CELERY_BEAT_SCHEDULE = { "clean_expired_models": { "task": "passbook.core.tasks.clean_expired_models", - "schedule": crontab(minute="*/5"), # Run every 5 minutes + "schedule": crontab(minute="*/5"), "options": {"queue": "passbook_scheduled"}, - } + }, + "db_backup": { + "task": "passbook.lib.tasks.backup.backup_database", + "schedule": crontab(minute=0, hour=0), + "options": {"queue": "passbook_scheduled"}, + }, } CELERY_TASK_CREATE_MISSING_QUEUES = True CELERY_TASK_DEFAULT_QUEUE = "passbook" @@ -445,6 +450,7 @@ for _app in INSTALLED_APPS: if DEBUG: INSTALLED_APPS.append("debug_toolbar") MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware") + CELERY_TASK_ALWAYS_EAGER = True INSTALLED_APPS.append("passbook.core.apps.PassbookCoreConfig")