root: implement db backups with monitored task, update docs

This commit is contained in:
Jens Langhammer 2020-10-19 22:17:47 +02:00
parent 2ed9a1dbe3
commit 4316ee4330
5 changed files with 50 additions and 61 deletions

View file

@ -6,6 +6,10 @@
### Backup ### Backup
!!! notice
Local backups are **enabled** by default, and will be run daily at 00:00
Local backups can be created by running the following command in your passbook installation directory Local backups can be created by running the following command in your passbook installation directory
``` ```
@ -14,15 +18,6 @@ docker-compose run --rm worker backup
This will dump the current database into the `./backups` folder. By defaults, the last 10 Backups are kept. This will dump the current database into the `./backups` folder. By defaults, the last 10 Backups are kept.
To schedule these backups, use the following snippet in a crontab
```
0 0 * * * bash -c "cd <passbook install location> && docker-compose run --rm worker backup" >/dev/null
```
!!! notice
passbook does support automatic backups on a schedule, however this is currently not recommended, as there is no way to monitor these scheduled tasks.
### Restore ### Restore
@ -42,11 +37,7 @@ After you've restored the backup, it is recommended to restart all services with
### S3 Configuration ### S3 Configuration
!!! notice #### Preparation
To trigger backups with S3 enabled, use the same commands as above.
#### S3 Preparation
passbook expects the bucket you select to already exist. The IAM User given to passbook should have the following permissions passbook expects the bucket you select to already exist. The IAM User given to passbook should have the following permissions
@ -101,11 +92,11 @@ Simply enable these options in your values.yaml file
```yaml ```yaml
# Enable Database Backups to S3 # Enable Database Backups to S3
backup: backup:
access_key: access-key accessKey: access-key
secret_key: secret-key secretKey: secret-key
bucket: s3-bucket bucket: s3-bucket
region: eu-central-1 region: eu-central-1
host: s3-host host: s3-host
``` ```
Afterwards, run a `helm upgrade` to update the ConfigMap. Because passbook-scheduled backups are not recommended currently, a Kubernetes CronJob is created that runs the backup daily. Afterwards, run a `helm upgrade` to update the ConfigMap. Backups are done automatically as above, at 00:00 every day.

View file

@ -1,42 +0,0 @@
{{- if .Values.backup }}
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: {{ include "passbook.fullname" . }}-backup
labels:
app.kubernetes.io/name: {{ include "passbook.name" . }}
helm.sh/chart: {{ include "passbook.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
schedule: "0 0 * * *"
jobTemplate:
spec:
template:
spec:
restartPolicy: Never
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
args: [server]
envFrom:
- configMapRef:
name: {{ include "passbook.fullname" . }}-config
prefix: PASSBOOK_
env:
- name: PASSBOOK_SECRET_KEY
valueFrom:
secretKeyRef:
name: "{{ include "passbook.fullname" . }}-secret-key"
key: "secret_key"
- name: PASSBOOK_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: "redis-password"
- name: PASSBOOK_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
key: "postgresql-password"
{{- end}}

View file

@ -0,0 +1,34 @@
"""Database backup task"""
from datetime import datetime
from io import StringIO
from botocore.exceptions import BotoCoreError, ClientError
from django.contrib.humanize.templatetags.humanize import naturaltime
from django.core import management
from structlog import get_logger
from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
from passbook.root.celery import CELERY_APP
LOGGER = get_logger()
@CELERY_APP.task(bind=True, base=MonitoredTask)
def backup_database(self: MonitoredTask): # pragma: no cover
"""Database backup"""
try:
start = datetime.now()
out = StringIO()
management.call_command("dbbackup", quiet=True, stdout=out)
self.set_status(
TaskResult(
TaskResultStatus.SUCCESSFUL,
[
f"Successfully finished database backup {naturaltime(start)}",
out.getvalue(),
],
)
)
LOGGER.info("Successfully backed up database.")
except (IOError, BotoCoreError, ClientError) as exc:
self.set_status(TaskResult(TaskResultStatus.ERROR).with_error(exc))

View file

@ -269,9 +269,14 @@ CELERY_TASK_SOFT_TIME_LIMIT = 600
CELERY_BEAT_SCHEDULE = { CELERY_BEAT_SCHEDULE = {
"clean_expired_models": { "clean_expired_models": {
"task": "passbook.core.tasks.clean_expired_models", "task": "passbook.core.tasks.clean_expired_models",
"schedule": crontab(minute="*/5"), # Run every 5 minutes "schedule": crontab(minute="*/5"),
"options": {"queue": "passbook_scheduled"}, "options": {"queue": "passbook_scheduled"},
} },
"db_backup": {
"task": "passbook.lib.tasks.backup.backup_database",
"schedule": crontab(minute=0, hour=0),
"options": {"queue": "passbook_scheduled"},
},
} }
CELERY_TASK_CREATE_MISSING_QUEUES = True CELERY_TASK_CREATE_MISSING_QUEUES = True
CELERY_TASK_DEFAULT_QUEUE = "passbook" CELERY_TASK_DEFAULT_QUEUE = "passbook"
@ -445,6 +450,7 @@ for _app in INSTALLED_APPS:
if DEBUG: if DEBUG:
INSTALLED_APPS.append("debug_toolbar") INSTALLED_APPS.append("debug_toolbar")
MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware") MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
CELERY_TASK_ALWAYS_EAGER = True
INSTALLED_APPS.append("passbook.core.apps.PassbookCoreConfig") INSTALLED_APPS.append("passbook.core.apps.PassbookCoreConfig")