core(major): add integrated database backup
This commit is contained in:
parent
8bdf12cff1
commit
4086252979
3
Pipfile
3
Pipfile
|
@ -36,6 +36,9 @@ signxml = "*"
|
|||
urllib3 = {extras = ["secure"],version = "*"}
|
||||
structlog = "*"
|
||||
pyuwsgi = "*"
|
||||
django-dbbackup = "*"
|
||||
boto3 = "*"
|
||||
django-storages = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "94b3d5140f0c31dac1fc77af75a0df30ae4fb0571bf6b7fcd722487c63dc1872"
|
||||
"sha256": "c319c982000c56e4db5e12296430ee12c1960c981309313e607fae9c0caad58f"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
@ -25,17 +25,17 @@
|
|||
},
|
||||
"asn1crypto": {
|
||||
"hashes": [
|
||||
"sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292",
|
||||
"sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f"
|
||||
"sha256:5abe83e773026162e4869f4ac16edf7554f661e8cc0bb6d2be3bc6915456731b",
|
||||
"sha256:8f3f9470d4ba7aa53afb00278dc26aac22dc3a0d4ed1335fd772f034e094401e"
|
||||
],
|
||||
"version": "==1.0.1"
|
||||
"version": "==1.1.0"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2",
|
||||
"sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396"
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
],
|
||||
"version": "==19.2.0"
|
||||
"version": "==19.3.0"
|
||||
},
|
||||
"billiard": {
|
||||
"hashes": [
|
||||
|
@ -44,6 +44,21 @@
|
|||
],
|
||||
"version": "==3.6.1.0"
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:4fbd407827bf79851bc86ac490ef31937e18d4133d464ea3774ee46d73b79251",
|
||||
"sha256:ad9b04038c0b4c45c08af626ecce9226e0d33dd8ef0bf473434b77d958df8c2c"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.9.249"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:0b9512a7cd56b2b903bd57c9818c242223ac7d9c10d7fa9e26616e365bb80b35",
|
||||
"sha256:fb0ba09bcd72872306729cedcd3da7f8afb7cd03efea82e3716db3890c04088e"
|
||||
],
|
||||
"version": "==1.12.249"
|
||||
},
|
||||
"celery": {
|
||||
"hashes": [
|
||||
"sha256:4c4532aa683f170f40bd76f928b70bc06ff171a959e06e71bf35f2f9d6031ef9",
|
||||
|
@ -61,36 +76,9 @@
|
|||
},
|
||||
"cffi": {
|
||||
"hashes": [
|
||||
"sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
|
||||
"sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
|
||||
"sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
|
||||
"sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
|
||||
"sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
|
||||
"sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
|
||||
"sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
|
||||
"sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
|
||||
"sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
|
||||
"sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
|
||||
"sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
|
||||
"sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
|
||||
"sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
|
||||
"sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
|
||||
"sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
|
||||
"sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
|
||||
"sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
|
||||
"sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
|
||||
"sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
|
||||
"sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
|
||||
"sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
|
||||
"sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
|
||||
"sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
|
||||
"sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
|
||||
"sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
|
||||
"sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
|
||||
"sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
|
||||
"sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
|
||||
"sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226"
|
||||
],
|
||||
"version": "==1.12.3"
|
||||
"version": "==1.13.0"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
|
@ -101,10 +89,10 @@
|
|||
},
|
||||
"cheroot": {
|
||||
"hashes": [
|
||||
"sha256:3ff64073efa35b39d5e107410f5c79664dc8c6c5990651e970740c80ab8878a8",
|
||||
"sha256:d523a1525258730026aa35b86c8c47c8d0e3892fb89f0f39157d4b32a50edf05"
|
||||
"sha256:42fa3e1f44d92b4784fd76621711b98f96aab66d504c383359a827286e896708",
|
||||
"sha256:89dc7aa6270b2e01860a6740769a99d039812e4cd6b1cab16e14ebd9f0ebb25c"
|
||||
],
|
||||
"version": "==8.1.0"
|
||||
"version": "==8.2.0"
|
||||
},
|
||||
"cherrypy": {
|
||||
"hashes": [
|
||||
|
@ -173,6 +161,13 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.4.0"
|
||||
},
|
||||
"django-dbbackup": {
|
||||
"hashes": [
|
||||
"sha256:9470e5d8bdaee4feb878b1b66c59eb9b27a131cccd648bf7cbfe70930acd4fc0"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"django-filters": {
|
||||
"hashes": [
|
||||
"sha256:1a9799a41106dc53ed894e952a24e8dee9b4fb37f010f22d178c09c90c61d711"
|
||||
|
@ -240,6 +235,14 @@
|
|||
"index": "pypi",
|
||||
"version": "==0.1.0"
|
||||
},
|
||||
"django-storages": {
|
||||
"hashes": [
|
||||
"sha256:87287b7ad2e789cd603373439994e1ac6f94d9dc2e5f8173d2a87aa3ed458bd9",
|
||||
"sha256:f3b3def96493d3ccde37b864cea376472baf6e8a596504b209278801c510b807"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.7.2"
|
||||
},
|
||||
"djangorestframework": {
|
||||
"hashes": [
|
||||
"sha256:5488aed8f8df5ec1d70f04b2114abc52ae6729748a176c453313834a9ee179c8",
|
||||
|
@ -247,6 +250,14 @@
|
|||
],
|
||||
"version": "==3.10.3"
|
||||
},
|
||||
"docutils": {
|
||||
"hashes": [
|
||||
"sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0",
|
||||
"sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827",
|
||||
"sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"
|
||||
],
|
||||
"version": "==0.15.2"
|
||||
},
|
||||
"drf-yasg": {
|
||||
"hashes": [
|
||||
"sha256:4cfec631880ae527a91ec7cd3241aea2f82189f59e2f089119aa687761afb227",
|
||||
|
@ -301,6 +312,13 @@
|
|||
],
|
||||
"version": "==2.10.3"
|
||||
},
|
||||
"jmespath": {
|
||||
"hashes": [
|
||||
"sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6",
|
||||
"sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c"
|
||||
],
|
||||
"version": "==0.9.4"
|
||||
},
|
||||
"kombu": {
|
||||
"hashes": [
|
||||
"sha256:389ba09e03b15b55b1a7371a441c894fd8121d174f5583bbbca032b9ea8c9edd",
|
||||
|
@ -557,6 +575,14 @@
|
|||
],
|
||||
"version": "==2.4.2"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
|
||||
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
|
||||
],
|
||||
"markers": "python_version >= '2.7'",
|
||||
"version": "==2.8.0"
|
||||
},
|
||||
"pytz": {
|
||||
"hashes": [
|
||||
"sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
|
||||
|
@ -623,10 +649,10 @@
|
|||
},
|
||||
"redis": {
|
||||
"hashes": [
|
||||
"sha256:98a22fb750c9b9bb46e75e945dc3f61d0ab30d06117cbb21ff9cd1d315fedd3b",
|
||||
"sha256:c504251769031b0dd7dd5cf786050a6050197c6de0d37778c80c08cb04ae8275"
|
||||
"sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62",
|
||||
"sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2"
|
||||
],
|
||||
"version": "==3.3.8"
|
||||
"version": "==3.3.11"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
|
@ -674,6 +700,13 @@
|
|||
"markers": "platform_python_implementation == 'CPython' and python_version < '3.8'",
|
||||
"version": "==0.2.0"
|
||||
},
|
||||
"s3transfer": {
|
||||
"hashes": [
|
||||
"sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d",
|
||||
"sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba"
|
||||
],
|
||||
"version": "==0.2.1"
|
||||
},
|
||||
"sentry-sdk": {
|
||||
"hashes": [
|
||||
"sha256:15e51e74b924180c98bcd636cb4634945b0a99a124d50b433c3a9dc6a582e8db",
|
||||
|
@ -744,6 +777,7 @@
|
|||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": null,
|
||||
"version": "==1.25.6"
|
||||
},
|
||||
"vine": {
|
||||
|
|
|
@ -11,4 +11,4 @@ LOGGER = get_logger()
|
|||
def clean_nonces():
|
||||
"""Remove expired nonces"""
|
||||
amount, _ = Nonce.objects.filter(expires__lt=now(), expiring=True).delete()
|
||||
LOGGER.debug("Deleted expired nonces", amount=amount)
|
||||
LOGGER.debug('Deleted expired nonces', amount=amount)
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
"""passbook misc tasks"""
|
||||
from django.core import management
|
||||
from structlog import get_logger
|
||||
|
||||
from passbook.root.celery import CELERY_APP
|
||||
|
||||
LOGGER = get_logger()
|
||||
|
||||
@CELERY_APP.task()
|
||||
def backup_database():
|
||||
"""Backup database"""
|
||||
management.call_command('dbbackup')
|
||||
LOGGER.info('Successfully backed up database.')
|
|
@ -24,6 +24,8 @@ from passbook import __version__
|
|||
from passbook.lib.config import CONFIG
|
||||
from passbook.lib.sentry import before_send
|
||||
|
||||
LOGGER = structlog.get_logger()
|
||||
|
||||
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
STATIC_ROOT = BASE_DIR + '/static'
|
||||
|
@ -213,7 +215,24 @@ CELERY_BROKER_URL = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.ho
|
|||
CELERY_RESULT_BACKEND = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}"
|
||||
f":6379/{CONFIG.y('redis.message_queue_db')}")
|
||||
|
||||
# Database backup
|
||||
if CONFIG.y('postgresql.backup'):
|
||||
INSTALLED_APPS += ['dbbackup']
|
||||
DBBACKUP_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
|
||||
|
||||
AWS_ACCESS_KEY_ID = CONFIG.y('postgresql.backup.access_key')
|
||||
AWS_SECRET_ACCESS_KEY = CONFIG.y('postgresql.backup.secret_key')
|
||||
AWS_STORAGE_BUCKET_NAME = CONFIG.y('postgresql.backup.bucket')
|
||||
AWS_S3_ENDPOINT_URL = CONFIG.y('postgresql.backup.host')
|
||||
AWS_DEFAULT_ACL = None
|
||||
LOGGER.info('Database backup is configured', host=CONFIG.y('postgresql.backup.host'))
|
||||
# Add automatic task to backup
|
||||
CELERY_BEAT_SCHEDULE['db_backup'] = {
|
||||
'task': 'passbook.lib.tasks.backup_database',
|
||||
'schedule': crontab(minute=0, hour=0) # Run every day, midnight
|
||||
}
|
||||
|
||||
# Sentry integration
|
||||
if not DEBUG:
|
||||
sentry_init(
|
||||
dsn="https://33cdbcb23f8b436dbe0ee06847410b67@sentry.beryju.org/3",
|
||||
|
|
Reference in New Issue