From 4086252979dbfa2ac458edd8fe68b2efff2d36c2 Mon Sep 17 00:00:00 2001 From: "Langhammer, Jens" Date: Tue, 15 Oct 2019 13:52:33 +0200 Subject: [PATCH] core(major): add integrated database backup --- Pipfile | 3 + Pipfile.lock | 118 ++++++++++++++++++++++++-------------- passbook/core/tasks.py | 2 +- passbook/lib/tasks.py | 13 +++++ passbook/root/settings.py | 19 ++++++ 5 files changed, 112 insertions(+), 43 deletions(-) create mode 100644 passbook/lib/tasks.py diff --git a/Pipfile b/Pipfile index 211c02a1b..95f93af7f 100644 --- a/Pipfile +++ b/Pipfile @@ -36,6 +36,9 @@ signxml = "*" urllib3 = {extras = ["secure"],version = "*"} structlog = "*" pyuwsgi = "*" +django-dbbackup = "*" +boto3 = "*" +django-storages = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 56e555694..4ff3861b8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "94b3d5140f0c31dac1fc77af75a0df30ae4fb0571bf6b7fcd722487c63dc1872" + "sha256": "c319c982000c56e4db5e12296430ee12c1960c981309313e607fae9c0caad58f" }, "pipfile-spec": 6, "requires": { @@ -25,17 +25,17 @@ }, "asn1crypto": { "hashes": [ - "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", - "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" + "sha256:5abe83e773026162e4869f4ac16edf7554f661e8cc0bb6d2be3bc6915456731b", + "sha256:8f3f9470d4ba7aa53afb00278dc26aac22dc3a0d4ed1335fd772f034e094401e" ], - "version": "==1.0.1" + "version": "==1.1.0" }, "attrs": { "hashes": [ - "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", - "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.2.0" + "version": "==19.3.0" }, "billiard": { "hashes": [ @@ -44,6 +44,21 @@ ], "version": "==3.6.1.0" }, + "boto3": { + "hashes": [ + "sha256:4fbd407827bf79851bc86ac490ef31937e18d4133d464ea3774ee46d73b79251", + "sha256:ad9b04038c0b4c45c08af626ecce9226e0d33dd8ef0bf473434b77d958df8c2c" + ], + "index": "pypi", + "version": "==1.9.249" + }, + "botocore": { + "hashes": [ + "sha256:0b9512a7cd56b2b903bd57c9818c242223ac7d9c10d7fa9e26616e365bb80b35", + "sha256:fb0ba09bcd72872306729cedcd3da7f8afb7cd03efea82e3716db3890c04088e" + ], + "version": "==1.12.249" + }, "celery": { "hashes": [ "sha256:4c4532aa683f170f40bd76f928b70bc06ff171a959e06e71bf35f2f9d6031ef9", @@ -61,36 +76,9 @@ }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" + "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226" ], - "version": "==1.12.3" + "version": "==1.13.0" }, "chardet": { "hashes": [ @@ -101,10 +89,10 @@ }, "cheroot": { "hashes": [ - "sha256:3ff64073efa35b39d5e107410f5c79664dc8c6c5990651e970740c80ab8878a8", - "sha256:d523a1525258730026aa35b86c8c47c8d0e3892fb89f0f39157d4b32a50edf05" + "sha256:42fa3e1f44d92b4784fd76621711b98f96aab66d504c383359a827286e896708", + "sha256:89dc7aa6270b2e01860a6740769a99d039812e4cd6b1cab16e14ebd9f0ebb25c" ], - "version": "==8.1.0" + "version": "==8.2.0" }, "cherrypy": { "hashes": [ @@ -173,6 +161,13 @@ "index": "pypi", "version": "==1.4.0" }, + "django-dbbackup": { + "hashes": [ + "sha256:9470e5d8bdaee4feb878b1b66c59eb9b27a131cccd648bf7cbfe70930acd4fc0" + ], + "index": "pypi", + "version": "==3.2.0" + }, "django-filters": { "hashes": [ "sha256:1a9799a41106dc53ed894e952a24e8dee9b4fb37f010f22d178c09c90c61d711" @@ -240,6 +235,14 @@ "index": "pypi", "version": "==0.1.0" }, + "django-storages": { + "hashes": [ + "sha256:87287b7ad2e789cd603373439994e1ac6f94d9dc2e5f8173d2a87aa3ed458bd9", + "sha256:f3b3def96493d3ccde37b864cea376472baf6e8a596504b209278801c510b807" + ], + "index": "pypi", + "version": "==1.7.2" + }, "djangorestframework": { "hashes": [ "sha256:5488aed8f8df5ec1d70f04b2114abc52ae6729748a176c453313834a9ee179c8", @@ -247,6 +250,14 @@ ], "version": "==3.10.3" }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, "drf-yasg": { "hashes": [ "sha256:4cfec631880ae527a91ec7cd3241aea2f82189f59e2f089119aa687761afb227", @@ -301,6 +312,13 @@ ], "version": "==2.10.3" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, "kombu": { "hashes": [ "sha256:389ba09e03b15b55b1a7371a441c894fd8121d174f5583bbbca032b9ea8c9edd", @@ -557,6 +575,14 @@ ], "version": "==2.4.2" }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "markers": "python_version >= '2.7'", + "version": "==2.8.0" + }, "pytz": { "hashes": [ "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", @@ -623,10 +649,10 @@ }, "redis": { "hashes": [ - "sha256:98a22fb750c9b9bb46e75e945dc3f61d0ab30d06117cbb21ff9cd1d315fedd3b", - "sha256:c504251769031b0dd7dd5cf786050a6050197c6de0d37778c80c08cb04ae8275" + "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", + "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" ], - "version": "==3.3.8" + "version": "==3.3.11" }, "requests": { "hashes": [ @@ -674,6 +700,13 @@ "markers": "platform_python_implementation == 'CPython' and python_version < '3.8'", "version": "==0.2.0" }, + "s3transfer": { + "hashes": [ + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" + ], + "version": "==0.2.1" + }, "sentry-sdk": { "hashes": [ "sha256:15e51e74b924180c98bcd636cb4634945b0a99a124d50b433c3a9dc6a582e8db", @@ -744,6 +777,7 @@ "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], "index": "pypi", + "markers": null, "version": "==1.25.6" }, "vine": { diff --git a/passbook/core/tasks.py b/passbook/core/tasks.py index 2966d4ee0..2b14af046 100644 --- a/passbook/core/tasks.py +++ b/passbook/core/tasks.py @@ -11,4 +11,4 @@ LOGGER = get_logger() def clean_nonces(): """Remove expired nonces""" amount, _ = Nonce.objects.filter(expires__lt=now(), expiring=True).delete() - LOGGER.debug("Deleted expired nonces", amount=amount) + LOGGER.debug('Deleted expired nonces', amount=amount) diff --git a/passbook/lib/tasks.py b/passbook/lib/tasks.py new file mode 100644 index 000000000..cc36f0ef5 --- /dev/null +++ b/passbook/lib/tasks.py @@ -0,0 +1,13 @@ +"""passbook misc tasks""" +from django.core import management +from structlog import get_logger + +from passbook.root.celery import CELERY_APP + +LOGGER = get_logger() + +@CELERY_APP.task() +def backup_database(): + """Backup database""" + management.call_command('dbbackup') + LOGGER.info('Successfully backed up database.') diff --git a/passbook/root/settings.py b/passbook/root/settings.py index 4bca8701c..57e693426 100644 --- a/passbook/root/settings.py +++ b/passbook/root/settings.py @@ -24,6 +24,8 @@ from passbook import __version__ from passbook.lib.config import CONFIG from passbook.lib.sentry import before_send +LOGGER = structlog.get_logger() + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) STATIC_ROOT = BASE_DIR + '/static' @@ -213,7 +215,24 @@ CELERY_BROKER_URL = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.ho CELERY_RESULT_BACKEND = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}" f":6379/{CONFIG.y('redis.message_queue_db')}") +# Database backup +if CONFIG.y('postgresql.backup'): + INSTALLED_APPS += ['dbbackup'] + DBBACKUP_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + AWS_ACCESS_KEY_ID = CONFIG.y('postgresql.backup.access_key') + AWS_SECRET_ACCESS_KEY = CONFIG.y('postgresql.backup.secret_key') + AWS_STORAGE_BUCKET_NAME = CONFIG.y('postgresql.backup.bucket') + AWS_S3_ENDPOINT_URL = CONFIG.y('postgresql.backup.host') + AWS_DEFAULT_ACL = None + LOGGER.info('Database backup is configured', host=CONFIG.y('postgresql.backup.host')) + # Add automatic task to backup + CELERY_BEAT_SCHEDULE['db_backup'] = { + 'task': 'passbook.lib.tasks.backup_database', + 'schedule': crontab(minute=0, hour=0) # Run every day, midnight + } + +# Sentry integration if not DEBUG: sentry_init( dsn="https://33cdbcb23f8b436dbe0ee06847410b67@sentry.beryju.org/3",