From f3ec1af69140462dade66b65fd284e9bda3b1544 Mon Sep 17 00:00:00 2001 From: glic3 Date: Mon, 4 May 2015 21:52:53 +0200 Subject: [PATCH] Added mailer --- TODO.md | 61 +++------ orchestra/admin/menu.py | 5 +- orchestra/contrib/accounts/settings.py | 3 +- orchestra/contrib/bills/settings.py | 3 +- orchestra/contrib/contacts/settings.py | 2 +- orchestra/contrib/databases/settings.py | 2 +- orchestra/contrib/domains/settings.py | 3 +- orchestra/contrib/issues/settings.py | 3 +- orchestra/contrib/lists/settings.py | 3 +- orchestra/contrib/mailboxes/settings.py | 3 +- orchestra/contrib/mailer/__init__.py | 0 orchestra/contrib/mailer/admin.py | 23 ++++ orchestra/contrib/mailer/backends.py | 30 +++++ orchestra/contrib/mailer/engine.py | 52 ++++++++ .../mailer/management/commands/send.py | 11 ++ orchestra/contrib/mailer/models.py | 68 ++++++++++ orchestra/contrib/mailer/settings.py | 6 + orchestra/contrib/mailer/tasks.py | 6 + orchestra/contrib/miscellaneous/settings.py | 2 +- orchestra/contrib/orchestration/settings.py | 2 +- orchestra/contrib/orders/settings.py | 2 +- orchestra/contrib/payments/settings.py | 2 +- orchestra/contrib/resources/settings.py | 2 +- orchestra/contrib/saas/__init__.py | 1 + orchestra/contrib/saas/apps.py | 15 +++ orchestra/contrib/saas/models.py | 24 +--- orchestra/contrib/saas/settings.py | 3 +- orchestra/contrib/saas/signals.py | 21 ++++ orchestra/contrib/services/settings.py | 2 +- orchestra/contrib/settings/__init__.py | 101 +++++++++++++++ orchestra/contrib/settings/admin.py | 2 +- orchestra/contrib/settings/apps.py | 21 ++++ orchestra/contrib/systemusers/settings.py | 2 +- orchestra/contrib/tasks/__init__.py | 109 +--------------- orchestra/contrib/tasks/beat.py | 2 +- orchestra/contrib/tasks/decorators.py | 100 +++++++++++++++ .../tasks/management/commands/runtask.py | 3 +- orchestra/contrib/tasks/settings.py | 8 +- orchestra/contrib/tasks/utils.py | 9 +- orchestra/contrib/vps/settings.py | 2 +- orchestra/contrib/webapps/__init__.py | 1 + orchestra/contrib/webapps/apps.py | 13 ++ orchestra/contrib/webapps/models.py | 24 +--- orchestra/contrib/webapps/settings.py | 3 +- orchestra/contrib/webapps/signals.py | 22 ++++ orchestra/contrib/websites/settings.py | 2 +- orchestra/settings.py | 119 +----------------- 47 files changed, 559 insertions(+), 344 deletions(-) create mode 100644 orchestra/contrib/mailer/__init__.py create mode 100644 orchestra/contrib/mailer/admin.py create mode 100644 orchestra/contrib/mailer/backends.py create mode 100644 orchestra/contrib/mailer/engine.py create mode 100644 orchestra/contrib/mailer/management/commands/send.py create mode 100644 orchestra/contrib/mailer/models.py create mode 100644 orchestra/contrib/mailer/settings.py create mode 100644 orchestra/contrib/mailer/tasks.py create mode 100644 orchestra/contrib/saas/apps.py create mode 100644 orchestra/contrib/saas/signals.py create mode 100644 orchestra/contrib/settings/apps.py create mode 100644 orchestra/contrib/tasks/decorators.py create mode 100644 orchestra/contrib/webapps/apps.py create mode 100644 orchestra/contrib/webapps/signals.py diff --git a/TODO.md b/TODO.md index c1d7ad1f..a449e93d 100644 --- a/TODO.md +++ b/TODO.md @@ -311,62 +311,36 @@ Replace celery by a custom solution? *priority: custom Thread backend *bulk: wrapper arround django-mailer to avoid loading django system -# Create a new virtualenv -python3 -mvenv env-django-orchestra -source env-django-orchestra/bin/activate -pip3 install django-orchestra==dev --allow-external django-orchestra --allow-unverified django-orchestra -# Install dependencies -sudo apt-get install python3.4-dev libxml2-dev libxslt1-dev libcrack2-dev -pip3 install -r https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt - -# Create an orchestra instance -orchestra-admin startproject panel -python3 panel/manage.py migrate accounts -python3 panel/manage.py migrate -python3 panel/manage.py runserver - -http://localhost:8000/admin/ - -setupcrontab +pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip -Collecting lxml==3.3.5 (from -r re (line 22)) - Downloading lxml-3.3.5.tar.gz (3.5MB) - 100% |################################| 3.5MB 60kB/s - Building lxml version 3.3.5. - Building without Cython. - ERROR: b'/bin/sh: 1: xslt-config: not found\n' - ** make sure the development packages of libxml2 and libxslt are installed ** - Using build configuration of libxslt - /usr/lib/python3.4/distutils/dist.py:260: UserWarning: Unknown distribution option: 'bugtrack_url' - warnings.warn(msg) + +# TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy + if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT: + import uwsgi + def uwsgi_beat(signum): + print "It's 5 o'clock of the first day of the month." + uwsgi.register_signal(99, '', uwsgi_beat) + uwsgi.add_timer(99, 60) +# TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi') +# SHip orchestra production-ready (no DEBUG etc) -# Setupcron -# uwsgi enable threads -# register signals in app ready() -# database_ready(): connect to the database or inspect django connection -# move Setting to contrib app __init__ -# cracklib vs crack -# remove system dependencies -# deprecate install_dependnecies in favour of only requirements.txt # import module and sed # if setting.value == default. remove -# TASKS_ENABLE_UWSGI_CRON # reload generic admin view ?redirect=http... -# inspecting django db connection for asserting db readines? +# inspecting django db connection for asserting db readines? or performing a query # wake up django mailer on send_mail # project settings modified copy of django's default project settings -# migrate accounts break on superuser insert because of orders signals: ready() + db_ready() +# all signals + accouns.register() services.register() on apps.py # if backend.async: don't join. # RELATED: domains.sync to ns3 make it async -# ngnix setup certificate from orchestra.contrib.tasks import task import time, sys @task(name='rata') @@ -378,10 +352,7 @@ Collecting lxml==3.3.5 (from -r re (line 22)) time.sleep(1) counter.apply_async(10, '/tmp/kakas') -# setup main systemuser on post_migrate SystemUser # Provide some fixtures with mocked data -don't make hard dependencies strict dependencies, fail when needed. -# on project_settings add debug settings but commented TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall @@ -389,8 +360,4 @@ TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix TODO mount the filesystem with "nosuid" option # execute Make after postfix update # wkhtmltopdf -> reportlab - - -# MAKE DEPENDENCIES OPTIONAL, check on deploy and warn that functionallity will not be available - - +# autoiscover modules on app.ready() diff --git a/orchestra/admin/menu.py b/orchestra/admin/menu.py index 93b11000..c5492253 100644 --- a/orchestra/admin/menu.py +++ b/orchestra/admin/menu.py @@ -95,15 +95,14 @@ def get_administration_items(): task = reverse('admin:djcelery_taskstate_changelist') periodic = reverse('admin:djcelery_periodictask_changelist') worker = reverse('admin:djcelery_workerstate_changelist') - childrens.append(items.MenuItem(_("Celery"), task, children=[ - items.MenuItem(_("Tasks"), task), + childrens.append(items.MenuItem(_("Tasks"), task, children=[ + items.MenuItem(_("Logs"), task), items.MenuItem(_("Periodic tasks"), periodic), items.MenuItem(_("Workers"), worker), ])) return childrens - class OrchestraMenu(Menu): template = 'admin/orchestra/menu.html' diff --git a/orchestra/contrib/accounts/settings.py b/orchestra/contrib/accounts/settings.py index f6ac48f4..7dfab167 100644 --- a/orchestra/contrib/accounts/settings.py +++ b/orchestra/contrib/accounts/settings.py @@ -1,7 +1,8 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN ACCOUNTS_TYPES = Setting('ACCOUNTS_TYPES', diff --git a/orchestra/contrib/bills/settings.py b/orchestra/contrib/bills/settings.py index 4bcb560d..0f131b2d 100644 --- a/orchestra/contrib/bills/settings.py +++ b/orchestra/contrib/bills/settings.py @@ -1,7 +1,8 @@ from django.conf import settings from django_countries import data -from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN BILLS_NUMBER_LENGTH = Setting('BILLS_NUMBER_LENGTH', diff --git a/orchestra/contrib/contacts/settings.py b/orchestra/contrib/contacts/settings.py index 64745f0d..111231f1 100644 --- a/orchestra/contrib/contacts/settings.py +++ b/orchestra/contrib/contacts/settings.py @@ -1,6 +1,6 @@ from django_countries import data -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting CONTACTS_DEFAULT_EMAIL_USAGES = Setting('CONTACTS_DEFAULT_EMAIL_USAGES', diff --git a/orchestra/contrib/databases/settings.py b/orchestra/contrib/databases/settings.py index 7975e115..fe69e8dc 100644 --- a/orchestra/contrib/databases/settings.py +++ b/orchestra/contrib/databases/settings.py @@ -1,6 +1,6 @@ from orchestra.core.validators import validate_hostname -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting DATABASES_TYPE_CHOICES = Setting('DATABASES_TYPE_CHOICES', diff --git a/orchestra/contrib/domains/settings.py b/orchestra/contrib/domains/settings.py index 2e9c6b73..e5808172 100644 --- a/orchestra/contrib/domains/settings.py +++ b/orchestra/contrib/domains/settings.py @@ -1,5 +1,6 @@ +from orchestra.contrib.settings import Setting from orchestra.core.validators import validate_ipv4_address, validate_ipv6_address, validate_ip_address -from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN from .validators import validate_zone_interval, validate_mx_record, validate_domain_name diff --git a/orchestra/contrib/issues/settings.py b/orchestra/contrib/issues/settings.py index 9346f8cc..902ba337 100644 --- a/orchestra/contrib/issues/settings.py +++ b/orchestra/contrib/issues/settings.py @@ -1,6 +1,7 @@ from django.core.validators import validate_email -from orchestra.settings import Setting, ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL ISSUES_SUPPORT_EMAILS = Setting('ISSUES_SUPPORT_EMAILS', diff --git a/orchestra/contrib/lists/settings.py b/orchestra/contrib/lists/settings.py index 725d2d3b..147b3d0e 100644 --- a/orchestra/contrib/lists/settings.py +++ b/orchestra/contrib/lists/settings.py @@ -1,4 +1,5 @@ -from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN LISTS_DOMAIN_MODEL = Setting('LISTS_DOMAIN_MODEL', diff --git a/orchestra/contrib/mailboxes/settings.py b/orchestra/contrib/mailboxes/settings.py index 9dd830b9..bd59983a 100644 --- a/orchestra/contrib/mailboxes/settings.py +++ b/orchestra/contrib/mailboxes/settings.py @@ -5,8 +5,9 @@ from django.utils.functional import lazy from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ +from orchestra.contrib.settings import Setting from orchestra.core.validators import validate_name -from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN _names = ('name', 'username',) diff --git a/orchestra/contrib/mailer/__init__.py b/orchestra/contrib/mailer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py new file mode 100644 index 00000000..70706e31 --- /dev/null +++ b/orchestra/contrib/mailer/admin.py @@ -0,0 +1,23 @@ +from django.contrib import admin + +from orchestra.admin.utils import admin_link + +from .models import Message, SMTPLog + + +class MessageAdmin(admin.ModelAdmin): + list_display = ( + 'id', 'state', 'priority', 'to_address', 'from_address', 'created_at', 'retries', 'last_retry' + ) + + +class SMTPLogAdmin(admin.ModelAdmin): + list_display = ( + 'id', 'message_link', 'result', 'date', 'log_message' + ) + + message_link = admin_link('message') + + +admin.site.register(Message, MessageAdmin) +admin.site.register(SMTPLog, SMTPLogAdmin) diff --git a/orchestra/contrib/mailer/backends.py b/orchestra/contrib/mailer/backends.py new file mode 100644 index 00000000..d6cd0f59 --- /dev/null +++ b/orchestra/contrib/mailer/backends.py @@ -0,0 +1,30 @@ +from django.core.mail.backends.base import BaseEmailBackend + +from .models import Message +from .tasks import send_message + + +class EmailBackend(BaseEmailBackend): + ''' + A wrapper that manages a queued SMTP system. + ''' + def send_messages(self, email_messages): + if not email_messages: + return + num_sent = 0 + for message in email_messages: + priority = message.extra_headers.get('X-Mail-Priority', Message.NORMAL) + if priority == Message.CRITICAL: + send_message(message).apply_async() + else: + content = message.message().as_string() + for to_email in message.recipients(): + message = Message.objects.create( + priority=priority, + to_address=to_email, + from_address=message.from_email, + subject=message.subject, + content=content, + ) + num_sent += 1 + return num_sent diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py new file mode 100644 index 00000000..9ed40abb --- /dev/null +++ b/orchestra/contrib/mailer/engine.py @@ -0,0 +1,52 @@ +import smtplib +from socket import error as SocketError + +from django.core.mail import get_connection +from django.utils.encoding import smart_str + +from .models import Message + + +def send_message(message, num, connection, bulk): + if num >= bulk: + connection.close() + connection = None + if connection is None: + # Reset connection + connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend') + connection.open() + error = None + try: + connection.connection.sendmail(message.from_address, [message.to_address], smart_str(message.content)) + except (SocketError, smtplib.SMTPSenderRefused, + smtplib.SMTPRecipientsRefused, + smtplib.SMTPAuthenticationError) as err: + message.defer() + error = err + else: + message.sent() + message.log(error) + + +def send_pending(bulk=100): + # TODO aquire lock + connection = None + num = 0 + for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'): + send_message(message, num, connection, bulk) + from django.utils import timezone + from . import settings + from datetime import timedelta + from django.db.models import Q + + now = timezone.now() + qs = Q() + for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS): + delta = timedelta(seconds=seconds) + qs = qs | Q(retries=retries, last_retry__lte=now-delta) + + for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'): + send_message(message, num, connection, bulk) + if connection is not None: + connection.close() + diff --git a/orchestra/contrib/mailer/management/commands/send.py b/orchestra/contrib/mailer/management/commands/send.py new file mode 100644 index 00000000..24d28c7a --- /dev/null +++ b/orchestra/contrib/mailer/management/commands/send.py @@ -0,0 +1,11 @@ +import json + +from django.core.management.base import BaseCommand, CommandError + +from ...engine import send_pending + +class Command(BaseCommand): + help = 'Runs Orchestra method.' + + def handle(self, *args, **options): + send_pending() diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py new file mode 100644 index 00000000..fc75969a --- /dev/null +++ b/orchestra/contrib/mailer/models.py @@ -0,0 +1,68 @@ +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from . import settings + + +class Message(models.Model): + QUEUED = 'QUEUED' + SENT = 'SENT' + DEFERRED = 'DEFERRED' + FAILED = 'FAILED' + STATES = ( + (QUEUED, _("Queued")), + (SENT, _("Sent")), + (DEFERRED, _("Deferred")), + (FAILED, _("Failes")), + ) + + CRITICAL = '0' + HIGH = '1' + NORMAL = '2' + LOW = '3' + PRIORITIES = ( + (CRITICAL, _("Critical (not queued)")), + (HIGH, _("High")), + (NORMAL, _("Normal")), + (LOW, _("Low")), + ) + + state = models.CharField(_("State"), max_length=16, choices=STATES, default=QUEUED) + priority = models.PositiveIntegerField(_("Priority"), choices=PRIORITIES, default=NORMAL) + to_address = models.CharField(max_length=256) + from_address = models.CharField(max_length=256) + subject = models.CharField(max_length=256) + content = models.TextField() + created_at = models.DateTimeField(auto_now_add=True) + retries = models.PositiveIntegerField(default=0) + last_retry = models.DateTimeField(auto_now=True) + + def defer(self): + self.state = self.DEFERRED + # Max tries + if self.retries >= len(settings.MAILER_DEFERE_SECONDS): + self.state = self.FAILED + self.save(update_fields=('state', 'retries')) + + def sent(self): + self.state = self.SENT + self.save(update_fields=('state',)) + + def log(self, error): + result = SMTPLog.SUCCESS + if error: + result= SMTPLog.FAILURE + self.logs.create(log_message=str(error), result=result) + + +class SMTPLog(models.Model): + SUCCESS = 'SUCCESS' + FAILURE = 'FAILURE' + RESULTS = ( + (SUCCESS, _("Success")), + (FAILURE, _("Failure")), + ) + message = models.ForeignKey(Message, editable=False, related_name='logs') + result = models.CharField(max_length=16, choices=RESULTS, default=SUCCESS) + date = models.DateTimeField(auto_now_add=True) + log_message = models.TextField() diff --git a/orchestra/contrib/mailer/settings.py b/orchestra/contrib/mailer/settings.py new file mode 100644 index 00000000..7c2290ee --- /dev/null +++ b/orchestra/contrib/mailer/settings.py @@ -0,0 +1,6 @@ +from orchestra.contrib.settings import Setting + + +MAILER_DEFERE_SECONDS = Setting('MAILER_DEFERE_SECONDS', + (300, 600, 60*60, 60*60*24), +) diff --git a/orchestra/contrib/mailer/tasks.py b/orchestra/contrib/mailer/tasks.py new file mode 100644 index 00000000..b5a4e70c --- /dev/null +++ b/orchestra/contrib/mailer/tasks.py @@ -0,0 +1,6 @@ +def send_message(): + pass + + +def cleanup_messages(): + pass diff --git a/orchestra/contrib/miscellaneous/settings.py b/orchestra/contrib/miscellaneous/settings.py index 77b448ba..bc0ef3e4 100644 --- a/orchestra/contrib/miscellaneous/settings.py +++ b/orchestra/contrib/miscellaneous/settings.py @@ -1,4 +1,4 @@ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting MISCELLANEOUS_IDENTIFIER_VALIDATORS = Setting('MISCELLANEOUS_IDENTIFIER_VALIDATORS', diff --git a/orchestra/contrib/orchestration/settings.py b/orchestra/contrib/orchestration/settings.py index aee38ed4..af93e90b 100644 --- a/orchestra/contrib/orchestration/settings.py +++ b/orchestra/contrib/orchestration/settings.py @@ -1,6 +1,6 @@ from os import path -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting ORCHESTRATION_OS_CHOICES = Setting('ORCHESTRATION_OS_CHOICES', diff --git a/orchestra/contrib/orders/settings.py b/orchestra/contrib/orders/settings.py index c3030c2c..6a59f0e3 100644 --- a/orchestra/contrib/orders/settings.py +++ b/orchestra/contrib/orders/settings.py @@ -1,4 +1,4 @@ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting ORDERS_BILLING_BACKEND = Setting('ORDERS_BILLING_BACKEND', diff --git a/orchestra/contrib/payments/settings.py b/orchestra/contrib/payments/settings.py index 215de3e6..41227e3a 100644 --- a/orchestra/contrib/payments/settings.py +++ b/orchestra/contrib/payments/settings.py @@ -1,4 +1,4 @@ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting from .. import payments diff --git a/orchestra/contrib/resources/settings.py b/orchestra/contrib/resources/settings.py index b85787b8..571df6aa 100644 --- a/orchestra/contrib/resources/settings.py +++ b/orchestra/contrib/resources/settings.py @@ -1,4 +1,4 @@ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting RESOURCES_TASK_BACKEND = Setting('RESOURCES_TASK_BACKEND', diff --git a/orchestra/contrib/saas/__init__.py b/orchestra/contrib/saas/__init__.py index e69de29b..0bc8af8b 100644 --- a/orchestra/contrib/saas/__init__.py +++ b/orchestra/contrib/saas/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.saas.apps.SaaSConfig' diff --git a/orchestra/contrib/saas/apps.py b/orchestra/contrib/saas/apps.py new file mode 100644 index 00000000..bed646b0 --- /dev/null +++ b/orchestra/contrib/saas/apps.py @@ -0,0 +1,15 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class SaaSConfig(AppConfig): + name = 'orchestra.contrib.saas' + verbose_name = 'Saas' + + def ready(self): + from . import signals + from .models import SaaS + services.register(SaaS) + + diff --git a/orchestra/contrib/saas/models.py b/orchestra/contrib/saas/models.py index ce64d7bf..2647be97 100644 --- a/orchestra/contrib/saas/models.py +++ b/orchestra/contrib/saas/models.py @@ -1,11 +1,9 @@ from django.db import models -from django.db.models.signals import pre_save, pre_delete -from django.dispatch import receiver from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from jsonfield import JSONField -from orchestra.core import services, validators +from orchestra.core import validators from .fields import VirtualDatabaseRelation from .services import SoftwareService @@ -73,23 +71,3 @@ class SaaS(models.Model): def set_password(self, password): self.password = password - - -services.register(SaaS) - - -# Admin bulk deletion doesn't call model.delete() -# So, signals are used instead of model method overriding - -@receiver(pre_save, sender=SaaS, dispatch_uid='saas.service.save') -def type_save(sender, *args, **kwargs): - instance = kwargs['instance'] - instance.service_instance.save() - -@receiver(pre_delete, sender=SaaS, dispatch_uid='saas.service.delete') -def type_delete(sender, *args, **kwargs): - instance = kwargs['instance'] - try: - instance.service_instance.delete() - except KeyError: - pass diff --git a/orchestra/contrib/saas/settings.py b/orchestra/contrib/saas/settings.py index 64f39bff..81a062b5 100644 --- a/orchestra/contrib/saas/settings.py +++ b/orchestra/contrib/saas/settings.py @@ -1,4 +1,5 @@ -from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN from .. import saas diff --git a/orchestra/contrib/saas/signals.py b/orchestra/contrib/saas/signals.py new file mode 100644 index 00000000..95a88eca --- /dev/null +++ b/orchestra/contrib/saas/signals.py @@ -0,0 +1,21 @@ +from django.db.models.signals import pre_save, pre_delete +from django.dispatch import receiver + +from .models import SaaS + + +# Admin bulk deletion doesn't call model.delete() +# So, signals are used instead of model method overriding + +@receiver(pre_save, sender=SaaS, dispatch_uid='saas.service.save') +def type_save(sender, *args, **kwargs): + instance = kwargs['instance'] + instance.service_instance.save() + +@receiver(pre_delete, sender=SaaS, dispatch_uid='saas.service.delete') +def type_delete(sender, *args, **kwargs): + instance = kwargs['instance'] + try: + instance.service_instance.delete() + except KeyError: + pass diff --git a/orchestra/contrib/services/settings.py b/orchestra/contrib/services/settings.py index ceda0b90..4a3f8634 100644 --- a/orchestra/contrib/services/settings.py +++ b/orchestra/contrib/services/settings.py @@ -1,6 +1,6 @@ from django.utils.translation import ugettext_lazy as _ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting SERVICES_SERVICE_TAXES = Setting('SERVICES_SERVICE_TAXES', diff --git a/orchestra/contrib/settings/__init__.py b/orchestra/contrib/settings/__init__.py index e69de29b..3be25c92 100644 --- a/orchestra/contrib/settings/__init__.py +++ b/orchestra/contrib/settings/__init__.py @@ -0,0 +1,101 @@ +import re +import sys +from collections import OrderedDict + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.utils.functional import Promise +from django.utils.translation import ugettext_lazy as _ + +from orchestra.core import validators +from orchestra.utils.python import import_class, format_exception + + +default_app_config = 'orchestra.contrib.settings.apps.SettingsConfig' + + +class Setting(object): + """ + Keeps track of the defined settings and provides extra batteries like value validation. + """ + conf_settings = settings + settings = OrderedDict() + + def __str__(self): + return self.name + + def __repr__(self): + value = str(self.value) + value = ("'%s'" if isinstance(value, str) else '%s') % value + return '<%s: %s>' % (self.name, value) + + def __new__(cls, name, default, help_text="", choices=None, editable=True, serializable=True, + multiple=False, validators=[], types=[], call_init=False): + if call_init: + return super(Setting, cls).__new__(cls) + cls.settings[name] = cls(name, default, help_text=help_text, choices=choices, editable=editable, + serializable=serializable, multiple=multiple, validators=validators, types=types, call_init=True) + return cls.get_value(name, default) + + def __init__(self, *args, **kwargs): + self.name, self.default = args + for name, value in kwargs.items(): + setattr(self, name, value) + self.value = self.get_value(self.name, self.default) + self.settings[name] = self + + @classmethod + def validate_choices(cls, value): + if not isinstance(value, (list, tuple)): + raise ValidationError("%s is not a valid choices." % str(value)) + for choice in value: + if not isinstance(choice, (list, tuple)) or len(choice) != 2: + raise ValidationError("%s is not a valid choice." % str(choice)) + value, verbose = choice + if not isinstance(verbose, (str, Promise)): + raise ValidationError("%s is not a valid verbose name." % value) + + @classmethod + def validate_import_class(cls, value): + try: + import_class(value) + except Exception as exc: + raise ValidationError(format_exception(exc)) + + @classmethod + def validate_model_label(cls, value): + from django.apps import apps + try: + apps.get_model(*value.split('.')) + except Exception as exc: + raise ValidationError(format_exception(exc)) + + @classmethod + def string_format_validator(cls, names, modulo=True): + def validate_string_format(value, names=names, modulo=modulo): + errors = [] + regex = r'%\(([^\)]+)\)' if modulo else r'{([^}]+)}' + for n in re.findall(regex, value): + if n not in names: + errors.append( + ValidationError('%s is not a valid format name.' % n) + ) + if errors: + raise ValidationError(errors) + return validate_string_format + + def validate_value(self, value): + if value: + validators.all_valid(value, self.validators) + valid_types = list(self.types) + if isinstance(self.default, (list, tuple)): + valid_types.extend([list, tuple]) + valid_types.append(type(self.default)) + if not isinstance(value, tuple(valid_types)): + raise ValidationError("%s is not a valid type (%s)." % + (type(value).__name__, ', '.join(t.__name__ for t in valid_types)) + ) + + @classmethod + def get_value(cls, name, default): + return getattr(cls.conf_settings, name, default) diff --git a/orchestra/contrib/settings/admin.py b/orchestra/contrib/settings/admin.py index 3d483690..3a503591 100644 --- a/orchestra/contrib/settings/admin.py +++ b/orchestra/contrib/settings/admin.py @@ -7,7 +7,7 @@ from django.views import generic from django.utils.translation import ngettext, ugettext_lazy as _ from orchestra.admin.dashboard import OrchestraIndexDashboard -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting from orchestra.utils import sys, paths from . import parser diff --git a/orchestra/contrib/settings/apps.py b/orchestra/contrib/settings/apps.py new file mode 100644 index 00000000..83f1b749 --- /dev/null +++ b/orchestra/contrib/settings/apps.py @@ -0,0 +1,21 @@ +from django.apps import AppConfig +from django.core.checks import register, Error +from django.core.exceptions import ValidationError + +from . import Setting + +class SettingsConfig(AppConfig): + name = 'orchestra.contrib.settings' + verbose_name = 'Settings' + + @register() + def check_settings(app_configs, **kwargs): + """ perfroms all the validation """ + messages = [] + for name, setting in Setting.settings.items(): + try: + setting.validate_value(setting.value) + except ValidationError as exc: + msg = "Error validating setting with value %s: %s" % (setting.value, str(exc)) + messages.append(Error(msg, obj=name, id='settings.E001')) + return messages diff --git a/orchestra/contrib/systemusers/settings.py b/orchestra/contrib/systemusers/settings.py index 8563131f..d2ac7c46 100644 --- a/orchestra/contrib/systemusers/settings.py +++ b/orchestra/contrib/systemusers/settings.py @@ -1,6 +1,6 @@ from django.utils.translation import ugettext_lazy as _ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting _names = ('user', 'username') diff --git a/orchestra/contrib/tasks/__init__.py b/orchestra/contrib/tasks/__init__.py index 509e3837..210d0ac6 100644 --- a/orchestra/contrib/tasks/__init__.py +++ b/orchestra/contrib/tasks/__init__.py @@ -1,107 +1,4 @@ -import traceback -from functools import partial, wraps, update_wrapper -from multiprocessing import Process -from uuid import uuid4 -from threading import Thread +import sys -from celery import shared_task as celery_shared_task -from celery import states -from celery.decorators import periodic_task as celery_periodic_task -from django.utils import timezone - -from orchestra.utils.db import close_connection -from orchestra.utils.python import AttrDict, OrderedSet - - -def get_id(): - return str(uuid4()) - - -def get_name(fn): - return '.'.join((fn.__module__, fn.__name__)) - - -def keep_state(fn): - """ logs task on djcelery's TaskState model """ - @wraps(fn) - def wrapper(task_id, name, *args, **kwargs): - from djcelery.models import TaskState - now = timezone.now() - state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args), - kwargs=str(kwargs), tstamp=now) - try: - result = fn(*args, **kwargs) - except Exception as exc: - state.state = states.FAILURE - state.traceback = traceback.format_exc() - state.runtime = (timezone.now()-now).total_seconds() - state.save() - return - # TODO send email - else: - state.state = states.SUCCESS - state.result = str(result) - state.runtime = (timezone.now()-now).total_seconds() - state.save() - return result - return wrapper - - -def apply_async(fn, name=None, method='thread'): - """ replaces celery apply_async """ - def inner(fn, name, method, *args, **kwargs): - task_id = get_id() - args = (task_id, name) + args - thread = method(target=fn, args=args, kwargs=kwargs) - thread.start() - # Celery API compat - thread.request = AttrDict(id=task_id) - return thread - - if name is None: - name = get_name(fn) - if method == 'thread': - method = Thread - elif method == 'process': - method = Process - else: - raise NotImplementedError("%s concurrency method is not supported." % method) - fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method) - fn.delay = fn.apply_async - return fn - - -def task(fn=None, **kwargs): - # TODO override this if 'celerybeat' in sys.argv ? - from . import settings - # register task - if fn is None: - name = kwargs.get('name', None) - if settings.TASKS_BACKEND in ('thread', 'process'): - def decorator(fn): - return apply_async(celery_shared_task(**kwargs)(fn), name=name) - return decorator - else: - return celery_shared_task(**kwargs) - fn = update_wraper(partial(celery_shared_task, fn)) - if settings.TASKS_BACKEND in ('thread', 'process'): - fn = update_wrapper(apply_async(fn), fn) - return fn - - -def periodic_task(fn=None, **kwargs): - from . import settings - # register task - if fn is None: - name = kwargs.get('name', None) - if settings.TASKS_BACKEND in ('thread', 'process'): - def decorator(fn): - return apply_async(celery_periodic_task(**kwargs)(fn), name=name) - return decorator - else: - return celery_periodic_task(**kwargs) - fn = update_wraper(celery_periodic_task(fn), fn) - if settings.TASKS_BACKEND in ('thread', 'process'): - name = kwargs.pop('name', None) - fn = update_wrapper(apply_async(fn, name), fn) - return fn +from . import settings +from .decorators import task, periodic_task, keep_state, apply_async diff --git a/orchestra/contrib/tasks/beat.py b/orchestra/contrib/tasks/beat.py index 53ed2369..81732b6d 100644 --- a/orchestra/contrib/tasks/beat.py +++ b/orchestra/contrib/tasks/beat.py @@ -5,7 +5,7 @@ from celery.schedules import crontab_parser as CrontabParser from django.utils import timezone from djcelery.models import PeriodicTask -from . import apply_async +from .decorators import apply_async def is_due(task, time=None): diff --git a/orchestra/contrib/tasks/decorators.py b/orchestra/contrib/tasks/decorators.py new file mode 100644 index 00000000..9deb77cb --- /dev/null +++ b/orchestra/contrib/tasks/decorators.py @@ -0,0 +1,100 @@ +import traceback +from functools import partial, wraps, update_wrapper +from multiprocessing import Process +from threading import Thread + +from celery import shared_task as celery_shared_task +from celery import states +from celery.decorators import periodic_task as celery_periodic_task +from django.utils import timezone + +from orchestra.utils.db import close_connection +from orchestra.utils.python import AttrDict, OrderedSet + +from .utils import get_name, get_id + + +def keep_state(fn): + """ logs task on djcelery's TaskState model """ + @wraps(fn) + def wrapper(task_id, name, *args, **kwargs): + from djcelery.models import TaskState + now = timezone.now() + state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args), + kwargs=str(kwargs), tstamp=now) + try: + result = fn(*args, **kwargs) + except Exception as exc: + state.state = states.FAILURE + state.traceback = traceback.format_exc() + state.runtime = (timezone.now()-now).total_seconds() + state.save() + return + # TODO send email + else: + state.state = states.SUCCESS + state.result = str(result) + state.runtime = (timezone.now()-now).total_seconds() + state.save() + return result + return wrapper + + +def apply_async(fn, name=None, method='thread'): + """ replaces celery apply_async """ + def inner(fn, name, method, *args, **kwargs): + task_id = get_id() + args = (task_id, name) + args + thread = method(target=fn, args=args, kwargs=kwargs) + thread.start() + # Celery API compat + thread.request = AttrDict(id=task_id) + return thread + + if name is None: + name = get_name(fn) + if method == 'thread': + method = Thread + elif method == 'process': + method = Process + else: + raise NotImplementedError("%s concurrency method is not supported." % method) + fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method) + fn.delay = fn.apply_async + return fn + + +def task(fn=None, **kwargs): + # TODO override this if 'celerybeat' in sys.argv ? + from . import settings + # register task + if fn is None: + name = kwargs.get('name', None) + if settings.TASKS_BACKEND in ('thread', 'process'): + def decorator(fn): + return apply_async(celery_shared_task(**kwargs)(fn), name=name) + return decorator + else: + return celery_shared_task(**kwargs) + fn = update_wraper(partial(celery_shared_task, fn)) + if settings.TASKS_BACKEND in ('thread', 'process'): + fn = update_wrapper(apply_async(fn), fn) + return fn + + +def periodic_task(fn=None, **kwargs): + from . import settings + # register task + if fn is None: + name = kwargs.get('name', None) + if settings.TASKS_BACKEND in ('thread', 'process'): + def decorator(fn): + return apply_async(celery_periodic_task(**kwargs)(fn), name=name) + return decorator + else: + return celery_periodic_task(**kwargs) + fn = update_wraper(celery_periodic_task(fn), fn) + if settings.TASKS_BACKEND in ('thread', 'process'): + name = kwargs.pop('name', None) + fn = update_wrapper(apply_async(fn, name), fn) + return fn diff --git a/orchestra/contrib/tasks/management/commands/runtask.py b/orchestra/contrib/tasks/management/commands/runtask.py index 73f855c9..a6a33f82 100644 --- a/orchestra/contrib/tasks/management/commands/runtask.py +++ b/orchestra/contrib/tasks/management/commands/runtask.py @@ -5,7 +5,8 @@ from django.core.management.base import BaseCommand, CommandError from django.utils import timezone from djcelery.models import PeriodicTask -from ... import keep_state, get_id, get_name +from ...decorators import keep_state +from ...utils import get_id, get_name class Command(BaseCommand): diff --git a/orchestra/contrib/tasks/settings.py b/orchestra/contrib/tasks/settings.py index 95bbb6d9..d6378856 100644 --- a/orchestra/contrib/tasks/settings.py +++ b/orchestra/contrib/tasks/settings.py @@ -1,4 +1,4 @@ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting TASKS_BACKEND = Setting('TASKS_BACKEND', @@ -9,3 +9,9 @@ TASKS_BACKEND = Setting('TASKS_BACKEND', ('celery', "Celery (with queue)"), ) ) + + +TASKS_ENABLE_UWSGI_CRON_BEAT = Setting('TASKS_ENABLE_UWSGI_CRON_BEAT', + False, + help_text="Not implemented.", +) diff --git a/orchestra/contrib/tasks/utils.py b/orchestra/contrib/tasks/utils.py index 21c74df3..19972696 100644 --- a/orchestra/contrib/tasks/utils.py +++ b/orchestra/contrib/tasks/utils.py @@ -1,9 +1,16 @@ import threading +from uuid import uuid4 from orchestra.utils.db import close_connection -# TODO import as_task +def get_id(): + return str(uuid4()) + + +def get_name(fn): + return '.'.join((fn.__module__, fn.__name__)) + def run(method, *args, **kwargs): async = kwargs.pop('async', True) diff --git a/orchestra/contrib/vps/settings.py b/orchestra/contrib/vps/settings.py index 1f3e1745..2466ff0d 100644 --- a/orchestra/contrib/vps/settings.py +++ b/orchestra/contrib/vps/settings.py @@ -1,4 +1,4 @@ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting VPS_TYPES = Setting('VPS_TYPES', diff --git a/orchestra/contrib/webapps/__init__.py b/orchestra/contrib/webapps/__init__.py index e69de29b..f08acd2d 100644 --- a/orchestra/contrib/webapps/__init__.py +++ b/orchestra/contrib/webapps/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.webapps.apps.WebAppsConfig' diff --git a/orchestra/contrib/webapps/apps.py b/orchestra/contrib/webapps/apps.py new file mode 100644 index 00000000..6f7c4650 --- /dev/null +++ b/orchestra/contrib/webapps/apps.py @@ -0,0 +1,13 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class WebAppsConfig(AppConfig): + name = 'orchestra.contrib.webapps' + verbose_name = 'Webapps' + + def ready(self): + from . import signals + from .models import WebApp + services.register(WebApp) diff --git a/orchestra/contrib/webapps/models.py b/orchestra/contrib/webapps/models.py index 9256919c..ff78f4bf 100644 --- a/orchestra/contrib/webapps/models.py +++ b/orchestra/contrib/webapps/models.py @@ -2,13 +2,11 @@ import os from collections import OrderedDict from django.db import models -from django.db.models.signals import pre_save, pre_delete -from django.dispatch import receiver from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from jsonfield import JSONField -from orchestra.core import validators, services +from orchestra.core import validators from orchestra.utils.functional import cached from . import settings @@ -121,23 +119,3 @@ class WebAppOption(models.Model): def clean(self): self.option_instance.validate() - - -services.register(WebApp) - - -# Admin bulk deletion doesn't call model.delete() -# So, signals are used instead of model method overriding - -@receiver(pre_save, sender=WebApp, dispatch_uid='webapps.type.save') -def type_save(sender, *args, **kwargs): - instance = kwargs['instance'] - instance.type_instance.save() - -@receiver(pre_delete, sender=WebApp, dispatch_uid='webapps.type.delete') -def type_delete(sender, *args, **kwargs): - instance = kwargs['instance'] - try: - instance.type_instance.delete() - except KeyError: - pass diff --git a/orchestra/contrib/webapps/settings.py b/orchestra/contrib/webapps/settings.py index 92fbe0a7..13628088 100644 --- a/orchestra/contrib/webapps/settings.py +++ b/orchestra/contrib/webapps/settings.py @@ -1,4 +1,5 @@ -from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN from .. import webapps diff --git a/orchestra/contrib/webapps/signals.py b/orchestra/contrib/webapps/signals.py new file mode 100644 index 00000000..ff849139 --- /dev/null +++ b/orchestra/contrib/webapps/signals.py @@ -0,0 +1,22 @@ +from django.db.models.signals import pre_save, pre_delete +from django.dispatch import receiver + +from .models import WebApp + + +# Admin bulk deletion doesn't call model.delete() +# So, signals are used instead of model method overriding + +@receiver(pre_save, sender=WebApp, dispatch_uid='webapps.type.save') +def type_save(sender, *args, **kwargs): + instance = kwargs['instance'] + instance.type_instance.save() + + +@receiver(pre_delete, sender=WebApp, dispatch_uid='webapps.type.delete') +def type_delete(sender, *args, **kwargs): + instance = kwargs['instance'] + try: + instance.type_instance.delete() + except KeyError: + pass diff --git a/orchestra/contrib/websites/settings.py b/orchestra/contrib/websites/settings.py index 597e8626..a8ccfb23 100644 --- a/orchestra/contrib/websites/settings.py +++ b/orchestra/contrib/websites/settings.py @@ -1,6 +1,6 @@ from django.utils.translation import ugettext_lazy as _ -from orchestra.settings import Setting +from orchestra.contrib.settings import Setting from .. import websites diff --git a/orchestra/settings.py b/orchestra/settings.py index 28ea9b84..dedab935 100644 --- a/orchestra/settings.py +++ b/orchestra/settings.py @@ -1,124 +1,7 @@ -import re -import sys -from collections import OrderedDict - -from django.conf import settings -from django.core.checks import register, Error -from django.core.exceptions import ValidationError, AppRegistryNotReady from django.core.validators import validate_email -from django.db.models import get_model -from django.utils.functional import Promise from django.utils.translation import ugettext_lazy as _ -from orchestra.utils.python import import_class, format_exception - -from .core import validators - - -class Setting(object): - """ - Keeps track of the defined settings and provides extra batteries like value validation. - """ - conf_settings = settings - settings = OrderedDict() - - def __str__(self): - return self.name - - def __repr__(self): - value = str(self.value) - value = ("'%s'" if isinstance(value, str) else '%s') % value - return '<%s: %s>' % (self.name, value) - - def __new__(cls, name, default, help_text="", choices=None, editable=True, serializable=True, - multiple=False, validators=[], types=[], call_init=False): - if call_init: - return super(Setting, cls).__new__(cls) - cls.settings[name] = cls(name, default, help_text=help_text, choices=choices, editable=editable, - serializable=serializable, multiple=multiple, validators=validators, types=types, call_init=True) - return cls.get_value(name, default) - - def __init__(self, *args, **kwargs): - self.name, self.default = args - for name, value in kwargs.items(): - setattr(self, name, value) - self.value = self.get_value(self.name, self.default) - self.settings[name] = self - - @classmethod - def validate_choices(cls, value): - if not isinstance(value, (list, tuple)): - raise ValidationError("%s is not a valid choices." % str(value)) - for choice in value: - if not isinstance(choice, (list, tuple)) or len(choice) != 2: - raise ValidationError("%s is not a valid choice." % str(choice)) - value, verbose = choice - if not isinstance(verbose, (str, Promise)): - raise ValidationError("%s is not a valid verbose name." % value) - - @classmethod - def validate_import_class(cls, value): - try: - import_class(value) - except ImportError as exc: - if "cannot import name 'settings'" in str(exc): - # circular dependency on init time - pass - except Exception as exc: - raise ValidationError(format_exception(exc)) - - @classmethod - def validate_model_label(cls, value): - try: - get_model(*value.split('.')) - except AppRegistryNotReady: - # circular dependency on init time - pass - except Exception as exc: - raise ValidationError(format_exception(exc)) - - @classmethod - def string_format_validator(cls, names, modulo=True): - def validate_string_format(value, names=names, modulo=modulo): - errors = [] - regex = r'%\(([^\)]+)\)' if modulo else r'{([^}]+)}' - for n in re.findall(regex, value): - if n not in names: - errors.append( - ValidationError('%s is not a valid format name.' % n) - ) - if errors: - raise ValidationError(errors) - return validate_string_format - - def validate_value(self, value): - if value: - validators.all_valid(value, self.validators) - valid_types = list(self.types) - if isinstance(self.default, (list, tuple)): - valid_types.extend([list, tuple]) - valid_types.append(type(self.default)) - if not isinstance(value, tuple(valid_types)): - raise ValidationError("%s is not a valid type (%s)." % - (type(value).__name__, ', '.join(t.__name__ for t in valid_types)) - ) - - @classmethod - def get_value(cls, name, default): - return getattr(cls.conf_settings, name, default) - - -@register() -def check_settings(app_configs, **kwargs): - """ perfroms all the validation """ - messages = [] - for name, setting in Setting.settings.items(): - try: - setting.validate_value(setting.value) - except ValidationError as exc: - msg = "Error validating setting with value %s: %s" % (setting.value, str(exc)) - messages.append(Error(msg, obj=name, id='settings.E001')) - return messages +from orchestra.contrib.settings import Setting ORCHESTRA_BASE_DOMAIN = Setting('ORCHESTRA_BASE_DOMAIN',