commit 0f26b5a8e0edcf75fa9c3932ee81b59742708641 Author: jorgepastorr Date: Sun Jul 9 07:51:51 2023 +0000 inicio 2023/07/09 diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0bca9ea5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.log +*.pot +*.pyc +*~ +*.save +.svn +local_settings.py +build +*.egg-info diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..293559d7 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,42 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..f2f93828 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..0459f8ae --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/orchestra.iml b/.idea/orchestra.iml new file mode 100644 index 00000000..f69c9a19 --- /dev/null +++ b/.idea/orchestra.iml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 00000000..6d3b336a --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,261 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + check_origin + updated_ + UPDATED_CHECK_ORIGIN + + + $PROJECT_DIR$/orchestra/contrib/lists + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + + def decorator(func, extra_context=extra_context, template=template, action_name=action_name, validatior=validator): + @wraps(func) + def inner(modeladmin, request, queryset, action_name=action_name, extra_context=extra_context, validator=validator): + if validator is not None: + try: + validator(queryset) + except ValidationError as e: + messages.error(request, '
'.join(e)) + return + # The user has already confirmed the action. + if request.POST.get('post') == 'generic_confirmation': + stay = func(modeladmin, request, queryset) + if not stay: + return + + opts = modeladmin.model._meta + app_label = opts.app_label + action_value = func.__name__ + + if len(queryset) == 1: + objects_name = force_str(opts.verbose_name) + obj = queryset.get() + else: + objects_name = force_str(opts.verbose_name_plural) + obj = None + if not action_name: + action_name = func.__name__ + context = { + 'title': _("Are you sure?"), + 'content_message': _("Are you sure you want to {action} the selected {item}?").format( + action=action_name, item=objects_name), + 'action_name': action_name.capitalize(), + 'action_value': action_value, + 'queryset': queryset, + 'opts': opts, + 'obj': obj, + 'app_label': app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + } + + if callable(extra_context): + extra_context = extra_context(modeladmin, request, queryset) + context.update(extra_context or {}) + if 'display_objects' not in context: + # Compute it only when necessary + context['display_objects'] = format_display_objects(modeladmin, request, queryset) + + # Display the confirmation page + return TemplateResponse(request, template, context) + return inner + return decorator diff --git a/orchestra/admin/forms.py b/orchestra/admin/forms.py new file mode 100644 index 00000000..f74df5e2 --- /dev/null +++ b/orchestra/admin/forms.py @@ -0,0 +1,228 @@ +import textwrap +from functools import partial + +from django import forms +from django.contrib.admin import helpers +from django.core import validators +from django.forms.models import modelformset_factory, BaseModelFormSet +from django.template import Template, Context +from django.utils.translation import gettext_lazy as _ + +from orchestra.forms.widgets import SpanWidget + +from ..core.validators import validate_password + + +class AdminFormMixin(object): + """ Provides a method for rendering a form just like in Django Admin """ + def as_admin(self): + prepopulated_fields = {} + fieldsets = [ + (None, { + 'fields': list(self.fields.keys()) + }), + ] + adminform = helpers.AdminForm(self, fieldsets, prepopulated_fields) + template = Template( + '{% for fieldset in adminform %}' + ' {% include "admin/includes/fieldset.html" %}' + '{% endfor %}' + ) + context = { + 'adminform': adminform + } + return template.render(Context(context)) + + +class AdminFormSet(BaseModelFormSet): + def as_admin(self): + template = Template(textwrap.dedent("""\ +
+
+ {{ formset.management_form }} +
+ {{ formset.non_form_errors.as_ul }} + + {% for form in formset.forms %} + {% if forloop.first %} + + {% for field in form.visible_fields %} + + {% endfor %} + + {% endif %} + + {% for field in form.visible_fields %} + + {% endfor %} + + {% endfor %} +
{{ field.label|capfirst }}
+ {# Include the hidden fields in the form #} + {% if forloop.first %} + {% for hidden in form.hidden_fields %} + {{ hidden }} + {% endfor %} + {% endif %} + {{ field.errors.as_ul }} + {{ field }} +
+
+
+
""") + ) + context = { + 'formset': self + } + return template.render(Context(context)) + + +class AdminPasswordChangeForm(forms.Form): + """ + A form used to change the password of a user in the admin interface. + """ + error_messages = { + 'password_mismatch': _("The two password fields didn't match."), + 'password_missing': _("No password has been provided."), + 'bad_hash': _("Invalid password format or unknown hashing algorithm."), + } + required_css_class = 'required' + password = forms.CharField(label=_("Password"), required=False, + widget=forms.TextInput(attrs={'size':'120'})) + password1 = forms.CharField(label=_("Password"), widget=forms.PasswordInput, + required=False, validators=[validate_password]) + password2 = forms.CharField(label=_("Password (again)"), widget=forms.PasswordInput, + required=False) + + def __init__(self, user, *args, **kwargs): + self.related = kwargs.pop('related', []) + self.raw = kwargs.pop('raw', False) + self.user = user + super().__init__(*args, **kwargs) + self.password_provided = False + for ix, rel in enumerate(self.related): + self.fields['password_%i' % ix] = forms.CharField(label=_("Password"), required=False, + widget=forms.TextInput(attrs={'size':'120'})) + setattr(self, 'clean_password_%i' % ix, partial(self.clean_password, ix=ix)) + self.fields['password1_%i' % ix] = forms.CharField(label=_("Password"), + widget=forms.PasswordInput, required=False) + self.fields['password2_%i' % ix] = forms.CharField(label=_("Password (again)"), + widget=forms.PasswordInput, required=False) + setattr(self, 'clean_password2_%i' % ix, partial(self.clean_password2, ix=ix)) + + def clean_password2(self, ix=''): + if ix != '': + ix = '_%i' % ix + password1 = self.cleaned_data.get('password1%s' % ix) + password2 = self.cleaned_data.get('password2%s' % ix) + if password1 and password2: + self.password_provided = True + if password1 != password2: + raise forms.ValidationError( + self.error_messages['password_mismatch'], + code='password_mismatch', + ) + elif password1 or password2: + self.password_provided = True + raise forms.ValidationError( + self.error_messages['password_mismatch'], + code='password_mismatch', + ) + return password2 + + def clean_password(self, ix=''): + if ix != '': + ix = '_%i' % ix + password = self.cleaned_data.get('password%s' % ix) + if password: + # lazy loading because of passlib + from django.contrib.auth.hashers import identify_hasher + self.password_provided = True + try: + identify_hasher(password) + except ValueError: + raise forms.ValidationError( + self.error_messages['bad_hash'], + code='bad_hash', + ) + return password + + def clean(self): + if not self.password_provided: + raise forms.ValidationError( + self.error_messages['password_missing'], + code='password_missing', + ) + + def save(self, commit=True): + """ + Saves the new password. + """ + field_name = 'password' if self.raw else 'password1' + password = self.cleaned_data[field_name] + if password: + if self.raw: + self.user.password = password + else: + self.user.set_password(password) + if commit: + try: + self.user.save(update_fields=['password']) + except ValueError: + # password is not a field but an attribute + self.user.save() # Trigger the backend + for ix, rel in enumerate(self.related): + password = self.cleaned_data['%s_%s' % (field_name, ix)] + if password: + if self.raw: + rel.password = password + else: + set_password = getattr(rel, 'set_password') + set_password(password) + if commit: + rel.save(update_fields=['password']) + return self.user + + def _get_changed_data(self): + data = super().changed_data + for name in self.fields.keys(): + if name not in data: + return [] + return ['password'] + changed_data = property(_get_changed_data) + + +class SendEmailForm(forms.Form): + email_from = forms.EmailField(label=_("From"), + widget=forms.TextInput(attrs={'size': '118'})) + to = forms.CharField(label="To", required=False) + extra_to = forms.CharField(label="To (extra)", required=False, + widget=forms.TextInput(attrs={'size': '118'})) + subject = forms.CharField(label=_("Subject"), + widget=forms.TextInput(attrs={'size': '118'})) + message = forms.CharField(label=_("Message"), + widget=forms.Textarea(attrs={'cols': 118, 'rows': 15})) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + initial = kwargs.get('initial') + if 'to' in initial: + self.fields['to'].widget = SpanWidget(original=initial['to']) + else: + self.fields.pop('to') + + def clean_comma_separated_emails(self, value): + clean_value = [] + for email in value.split(','): + email = email.strip() + if email: + try: + validators.validate_email(email) + except validators.ValidationError: + raise validators.ValidationError("Comma separated email addresses.") + clean_value.append(email) + return clean_value + + def clean_extra_to(self): + extra_to = self.cleaned_data['extra_to'] + return self.clean_comma_separated_emails(extra_to) diff --git a/orchestra/admin/html.py b/orchestra/admin/html.py new file mode 100644 index 00000000..208e0b34 --- /dev/null +++ b/orchestra/admin/html.py @@ -0,0 +1,20 @@ +from django.utils.safestring import mark_safe + + +MONOSPACE_FONTS = ('Consolas,Monaco,Lucida Console,Liberation Mono,DejaVu Sans Mono,' + 'Bitstream Vera Sans Mono,Courier New,monospace') + + +def monospace_format(text): + style="font-family:%s;padding-left:110px;white-space:pre-wrap;" % MONOSPACE_FONTS + return mark_safe('
%s
' % (style, text)) + + +def code_format(text, language='bash'): + from pygments import highlight + from pygments.lexers import get_lexer_by_name + from pygments.formatters import HtmlFormatter + lexer = get_lexer_by_name(language, stripall=True) + formatter = HtmlFormatter(linenos=True) + code = highlight(text, lexer, formatter) + return mark_safe('
%s
' % code) diff --git a/orchestra/admin/menu.py b/orchestra/admin/menu.py new file mode 100644 index 00000000..f849829e --- /dev/null +++ b/orchestra/admin/menu.py @@ -0,0 +1,100 @@ +from copy import deepcopy + +from admin_tools.menu import items, Menu +from django.urls import reverse +from django.utils.text import capfirst +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import services, accounts, administration + + +def api_link(context): + """ Dynamically generates API related URL """ + if 'opts' in context: + opts = context['opts'] + elif 'cl' in context: + opts = context['cl'].opts + else: + return reverse('api-root') + if 'object_id' in context: + object_id = context['object_id'] + try: + return reverse('%s-detail' % opts.model_name, args=[object_id]) + except: + return reverse('api-root') + try: + return reverse('%s-list' % opts.model_name) + except: + return reverse('api-root') + + +def process_registry(register): + def get_item(model, options, name=None): + if name is None: + name = capfirst(options.get('verbose_name_plural')) + if isinstance(model, str): + url = reverse('admin:'+model) + else: + opts = model._meta + url = reverse('admin:{}_{}_changelist'.format( + opts.app_label, opts.model_name) + ) + item = items.MenuItem(name, url) + item.options = options + return item + + childrens = {} + for model, options in register.get().items(): + if options.get('menu', True): + parent = options.get('parent') + if parent: + name = capfirst(model._meta.app_label) + parent_item = childrens.get(parent) + if parent_item: + if not parent_item.children: + parent_item.children.append(deepcopy(parent_item)) + parent_item.title = name + else: + parent_item = get_item(parent, register[parent], name=name) + parent_item.children = [] + parent_item.children.append(get_item(model, options)) + childrens[parent] = parent_item + elif model not in childrens: + childrens[model] = get_item(model, options) + else: + childrens[model].children.insert(0, get_item(model, options)) + return sorted(childrens.values(), key=lambda i: i.title) + + +class OrchestraMenu(Menu): + template = 'admin/orchestra/menu.html' + + def init_with_context(self, context): + self.children = [ +# items.MenuItem( +# mark_safe('{site_name} v{version}'.format( +# site_name=force_str(settings.SITE_VERBOSE_NAME), +# version_style="text-transform:none; float:none; font-size:smaller; background:none;", +# version=get_version())), +# reverse('admin:index') +# ), +# items.MenuItem( +# _('Dashboard'), +# reverse('admin:index') +# ), +# items.Bookmarks(), + items.MenuItem( + _("Services"), + children=process_registry(services) + ), + items.MenuItem( + _("Accounts"), + reverse('admin:accounts_account_changelist'), + children=process_registry(accounts) + ), + items.MenuItem( + _("Administration"), + children=process_registry(administration) + ), + items.MenuItem("API", api_link(context)), + ] diff --git a/orchestra/admin/options.py b/orchestra/admin/options.py new file mode 100644 index 00000000..3849bc20 --- /dev/null +++ b/orchestra/admin/options.py @@ -0,0 +1,339 @@ +from urllib import parse + +from django import forms +from django.urls import re_path as url +from django.contrib import admin, messages +from django.contrib.admin.options import IS_POPUP_VAR +from django.contrib.admin.utils import unquote +from django.contrib.auth import update_session_auth_hash +from django.core.exceptions import PermissionDenied +from django.http import HttpResponseRedirect, Http404, HttpResponse +from django.forms.models import BaseInlineFormSet +from django.shortcuts import get_object_or_404 +from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator +from django.utils.encoding import force_str +from django.utils.html import escape +from django.utils.translation import gettext_lazy as _ +from django.views.decorators.debug import sensitive_post_parameters + +from orchestra.models.utils import has_db_field + +from ..utils.python import random_ascii, pairwise + +from .forms import AdminPasswordChangeForm +#, AdminRawPasswordChangeForm +#from django.contrib.auth.forms import AdminPasswordChangeForm +from .utils import action_to_view + + +sensitive_post_parameters_m = method_decorator(sensitive_post_parameters()) + + +class ChangeListDefaultFilter(object): + """ + Enables support for default filtering on admin change list pages + Your model admin class should define an default_changelist_filters attribute + default_changelist_filters = (('my_nodes', 'True'),) + """ + default_changelist_filters = () + + def changelist_view(self, request, extra_context=None): +# defaults = [] +# for key, value in self.default_changelist_filters: +# set_url_query(request, key, value) +# defaults.append(key) +# # hack response cl context in order to hook default filter awaearness +# # into search_form.html template +# response = super(ChangeListDefaultFilter, self).changelist_view(request, extra_context) +# if hasattr(response, 'context_data') and 'cl' in response.context_data: +# response.context_data['cl'].default_changelist_filters = defaults +# return response + querystring = request.META['QUERY_STRING'] + querydict = parse.parse_qs(querystring) + redirect = False + for field, value in self.default_changelist_filters: + if field not in querydict: + redirect = True + querydict[field] = value + if redirect: + querystring = parse.urlencode(querydict, doseq=True) + return HttpResponseRedirect(request.path + '?%s' % querystring) + return super(ChangeListDefaultFilter, self).changelist_view(request, extra_context=extra_context) + + +class AtLeastOneRequiredInlineFormSet(BaseInlineFormSet): + def clean(self): + """Check that at least one service has been entered.""" + super(AtLeastOneRequiredInlineFormSet, self).clean() + if any(self.errors): + return + if not any(cleaned_data and not cleaned_data.get('DELETE', False) + for cleaned_data in self.cleaned_data): + raise forms.ValidationError('At least one item required.') + + +class EnhaceSearchMixin(object): + def lookup_allowed(self, lookup, value): + """ allows any lookup """ + if 'password' in lookup: + return False + return True + + def get_search_results(self, request, queryset, search_term): + """ allows to specify field : """ + search_fields = self.get_search_fields(request) + if '=' in search_term: + fields = {field.split('__')[0]: field for field in search_fields} + new_search_term = [] + for part in search_term.split(): + field = None + if '=' in part: + field, term = part.split('=') + kwarg = '%s__icontains' + c_term = term + if term.startswith(('"', "'")) and term.endswith(('"', "'")): + c_term = term[1:-1] + kwarg = '%s__iexact' + if field in fields: + queryset = queryset.filter(**{kwarg % fields[field]: c_term}) + else: + new_search_term.append('='.join((field, term))) + else: + new_search_term.append(part) + search_term = ' '.join(new_search_term) + return super(EnhaceSearchMixin, self).get_search_results(request, queryset, search_term) + + +class ChangeViewActionsMixin(object): + """ Makes actions visible on the admin change view page. """ + change_view_actions = () + change_form_template = 'orchestra/admin/change_form.html' + + def get_urls(self): + """Returns the additional urls for the change view links""" + urls = super(ChangeViewActionsMixin, self).get_urls() + admin_site = self.admin_site + opts = self.model._meta + new_urls = [] + for action in self.get_change_view_actions(): + new_urls.append( + url('^(\d+)/%s/$' % action.url_name, + admin_site.admin_view(action), + name='%s_%s_%s' % (opts.app_label, opts.model_name, action.url_name) + ) + ) + return new_urls + urls + + def get_change_view_actions(self, obj=None): + """ allow customization on modelamdin """ + views = [] + for action in self.change_view_actions: + if isinstance(action, str): + action = getattr(self, action) + view = action_to_view(action, self) + view.url_name = getattr(action, 'url_name', action.__name__) + tool_description = getattr(action, 'tool_description', '') + if not tool_description: + tool_description = getattr(action, 'short_description', + view.url_name.capitalize().replace('_', ' ')) + if hasattr(tool_description, '__call__'): + tool_description = tool_description(obj) + view.tool_description = tool_description + view.css_class = getattr(action, 'css_class', 'historylink') + view.help_text = getattr(action, 'help_text', '') + view.hidden = getattr(action, 'hidden', False) + views.append(view) + return views + + def change_view(self, request, object_id, **kwargs): + if kwargs.get('extra_context', None) is None: + kwargs['extra_context'] = {} + obj = self.get_object(request, unquote(object_id)) + kwargs['extra_context']['object_tools_items'] = [ + action.__dict__ for action in self.get_change_view_actions(obj) if not action.hidden + ] + return super().change_view(request, object_id, **kwargs) + + +class ChangeAddFieldsMixin(object): + """ Enables to specify different set of fields for change and add views """ + add_fields = () + add_fieldsets = () + add_form = None + add_prepopulated_fields = {} + change_readonly_fields = () + change_form = None + add_inlines = None + + def get_prepopulated_fields(self, request, obj=None): + if not obj: + return super(ChangeAddFieldsMixin, self).get_prepopulated_fields(request, obj) + return {} + + def get_change_readonly_fields(self, request, obj=None): + return self.change_readonly_fields + + def get_readonly_fields(self, request, obj=None): + fields = super(ChangeAddFieldsMixin, self).get_readonly_fields(request, obj) + if obj: + return fields + self.get_change_readonly_fields(request, obj) + return fields + + def get_fieldsets(self, request, obj=None): + if not obj: + if self.add_fieldsets: + return self.add_fieldsets + elif self.add_fields: + return [(None, {'fields': self.add_fields})] + return super(ChangeAddFieldsMixin, self).get_fieldsets(request, obj) + + def get_inline_instances(self, request, obj=None): + """ add_inlines and inline.parent_object """ + if obj: + self.inlines = type(self).inlines + else: + self.inlines = self.inlines if self.add_inlines is None else self.add_inlines + inlines = super(ChangeAddFieldsMixin, self).get_inline_instances(request, obj) + for inline in inlines: + inline.parent_object = obj + return inlines + + def get_form(self, request, obj=None, **kwargs): + """ Use special form during user creation """ + defaults = {} + if obj is None: + if self.add_form: + defaults['form'] = self.add_form + else: + if self.change_form: + defaults['form'] = self.change_form + defaults.update(kwargs) + return super(ChangeAddFieldsMixin, self).get_form(request, obj, **defaults) + + +class ExtendedModelAdmin(ChangeViewActionsMixin, + ChangeAddFieldsMixin, + ChangeListDefaultFilter, + EnhaceSearchMixin, + admin.ModelAdmin): + list_prefetch_related = None + + def get_queryset(self, request): + qs = super(ExtendedModelAdmin, self).get_queryset(request) + if self.list_prefetch_related: + qs = qs.prefetch_related(*self.list_prefetch_related) + return qs + + def get_object(self, request, object_id, from_field=None): + obj = super(ExtendedModelAdmin, self).get_object(request, object_id, from_field) + if obj is None: + opts = self.model._meta + raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % { + 'name': force_str(opts.verbose_name), 'key': escape(object_id)}) + return obj + + +class ChangePasswordAdminMixin(object): + change_password_form = AdminPasswordChangeForm + change_user_password_template = 'admin/orchestra/change_password.html' + + def get_urls(self): + opts = self.model._meta + info = opts.app_label, opts.model_name + return [ + url(r'^(\d+)/password/$', + self.admin_site.admin_view(self.change_password), + name='%s_%s_change_password' % info), + url(r'^(\d+)/hash/$', + self.admin_site.admin_view(self.show_hash), + name='%s_%s_show_hash' % info) + ] + super().get_urls() + + def get_change_password_username(self, obj): + return str(obj) + + @sensitive_post_parameters_m + def change_password(self, request, id, form_url=''): + if not self.has_change_permission(request): + raise PermissionDenied + # TODO use this insetad of self.get_object(), in other places + obj = get_object_or_404(self.get_queryset(request), pk=id) + raw = request.GET.get('raw', '0') == '1' + can_raw = has_db_field(obj, 'password') + if raw and not can_raw: + raise TypeError("%s has no password db field for raw password edditing." % obj) + related = [] + for obj_name_attr in ('username', 'name', 'hostname'): + try: + obj_name = getattr(obj, obj_name_attr) + except AttributeError: + pass + else: + break + if hasattr(obj, 'account'): + account = obj.account + if obj.account.username == obj_name: + related.append(obj.account) + else: + account = obj + if account.username == obj_name: + for rel in account.get_related_passwords(db_field=raw): + if not isinstance(obj, type(rel)): + related.append(rel) + + if request.method == 'POST': + form = self.change_password_form(obj, request.POST, related=related, raw=raw) + if form.is_valid(): + form.save() + self.log_change(request, obj, _("Password changed.")) + msg = _('Password changed successfully.') + messages.success(request, msg) + update_session_auth_hash(request, form.user) # This is safe + return HttpResponseRedirect('..') + else: + form = self.change_password_form(obj, related=related, raw=raw) + + fieldsets = [ + (obj._meta.verbose_name.capitalize(), { + 'classes': ('wide',), + 'fields': ('password',) if raw else ('password1', 'password2'), + }), + ] + for ix, rel in enumerate(related): + fieldsets.append((rel._meta.verbose_name.capitalize(), { + 'classes': ('wide',), + 'fields': ('password_%i' % ix,) if raw else ('password1_%i' % ix, 'password2_%i' % ix) + })) + + obj_username = self.get_change_password_username(obj) + adminForm = admin.helpers.AdminForm(form, fieldsets, {}) + context = { + 'title': _('Change password: %s') % obj_username, + 'adminform': adminForm, + 'raw': raw, + 'can_raw': can_raw, + 'errors': admin.helpers.AdminErrorList(form, []), + 'form_url': form_url, + 'is_popup': (IS_POPUP_VAR in request.POST or + IS_POPUP_VAR in request.GET), + 'add': True, + 'change': False, + 'has_delete_permission': False, + 'has_change_permission': True, + 'has_absolute_url': False, + 'opts': self.model._meta, + 'original': obj, + 'obj_username': obj_username, + 'save_as': False, + 'show_save': True, + 'password': random_ascii(10), + } + context.update(admin.site.each_context(request)) + return TemplateResponse(request, self.change_user_password_template, context) + + def show_hash(self, request, id): + if not request.user.is_superuser: + raise PermissionDenied + obj = get_object_or_404(self.get_queryset(request), pk=id) + return HttpResponse(obj.password) diff --git a/orchestra/admin/utils.py b/orchestra/admin/utils.py new file mode 100644 index 00000000..e38ceb72 --- /dev/null +++ b/orchestra/admin/utils.py @@ -0,0 +1,185 @@ +import datetime +import importlib +import inspect +from functools import wraps + +from django.conf import settings +from django.contrib import admin +from django.core.exceptions import ObjectDoesNotExist +from django.urls import reverse, NoReverseMatch +from django.db import models +from django.shortcuts import redirect +from django.utils import timezone +from django.utils.html import escape, format_html +from django.utils.safestring import mark_safe + +from orchestra.models.utils import get_field_value +from orchestra.utils import humanize + +from .decorators import admin_field +from .html import monospace_format, code_format + + +def get_modeladmin(model, import_module=True): + """ returns the modeladmin registred for model """ + for k,v in admin.site._registry.items(): + if k is model: + return v + if import_module: + # Sometimes the admin module is not yet imported + app_label = model._meta.app_label + for app in settings.INSTALLED_APPS: + if app.endswith(app_label): + app_label = app + importlib.import_module('%s.%s' % (app_label, 'admin')) + return get_modeladmin(model, import_module=False) + + +def insertattr(model, name, value): + """ Inserts attribute to a modeladmin """ + modeladmin = None + if issubclass(model, models.Model): + modeladmin = get_modeladmin(model) + modeladmin_class = type(modeladmin) + elif not inspect.isclass(model): + modeladmin = model + modeladmin_class = type(modeladmin) + else: + modeladmin_class = model + # Avoid inlines defined on parent class be shared between subclasses + # Seems that if we use tuples they are lost in some conditions like changing + # the tuple in modeladmin.__init__ + if not getattr(modeladmin_class, name): + setattr(modeladmin_class, name, []) + setattr(modeladmin_class, name, list(getattr(modeladmin_class, name))+[value]) + if modeladmin: + # make sure class and object share the same attribute, to avoid wierd bugs + setattr(modeladmin, name, getattr(modeladmin_class, name)) + + +def wrap_admin_view(modeladmin, view): + """ Add admin authentication to view """ + @wraps(view) + def wrapper(*args, **kwargs): + return modeladmin.admin_site.admin_view(view)(*args, **kwargs) + return wrapper + + +def set_url_query(request, key, value): + """ set default filters for changelist_view """ + if key not in request.GET: + request_copy = request.GET.copy() + if callable(value): + value = value(request) + request_copy[key] = value + request.GET = request_copy + request.META['QUERY_STRING'] = request.GET.urlencode() + + +def action_to_view(action, modeladmin): + """ Converts modeladmin action to view function """ + @wraps(action) + def action_view(request, object_id=1, modeladmin=modeladmin, action=action): + queryset = modeladmin.model.objects.filter(pk=object_id) + response = action(modeladmin, request, queryset) + if not response: + opts = modeladmin.model._meta + url = 'admin:%s_%s_change' % (opts.app_label, opts.model_name) + return redirect(url, object_id) + return response + return action_view + + +def change_url(obj): + if obj is not None: + cls = type(obj) + opts = obj._meta + if cls is models.DEFERRED: + opts = cls.__base__._meta + view_name = 'admin:%s_%s_change' % (opts.app_label, opts.model_name) + return reverse(view_name, args=(obj.pk,)) + raise NoReverseMatch + + +@admin_field +def admin_link(*args, **kwargs): + instance = args[-1] + if kwargs['field'] in ('id', 'pk', '__str__'): + obj = instance + else: + try: + obj = get_field_value(instance, kwargs['field']) + except ObjectDoesNotExist: + return '---' + if not getattr(obj, 'pk', None): + return '---' + display_ = kwargs.get('display') + if display_: + display_ = getattr(obj, display_, display_) + else: + display_ = obj + try: + url = change_url(obj) + except NoReverseMatch: + # Does not has admin + return str(display_) + extra = '' + if kwargs['popup']: + extra = mark_safe('onclick="return showAddAnotherPopup(this);"') + title = "Change %s" % obj._meta.verbose_name + return format_html('{}', url, title, extra, display_) + + +@admin_field +def admin_colored(*args, **kwargs): + instance = args[-1] + field = kwargs['field'] + value = escape(get_field_value(instance, field)) + color = kwargs.get('colors', {}).get(value, 'black') + value = getattr(instance, 'get_%s_display' % field)().upper() + colored_value = '%s' % (color, value) + if kwargs.get('bold', True): + colored_value = '%s' % colored_value + return mark_safe(colored_value) + + +@admin_field +def admin_date(*args, **kwargs): + instance = args[-1] + date = get_field_value(instance, kwargs['field']) + if not date: + return kwargs.get('default', '') + if isinstance(date, datetime.datetime): + natural = humanize.naturaldatetime(date) + else: + natural = humanize.naturaldate(date) + if hasattr(date, 'hour'): + date = timezone.localtime(date) + date = date.strftime("%Y-%m-%d %H:%M:%S %Z") + else: + date = date.strftime("%Y-%m-%d") + return format_html('{1}', date, natural) + + +def get_object_from_url(modeladmin, request): + try: + object_id = int(request.path.split('/')[-3]) + except ValueError: + return None + else: + return modeladmin.model.objects.get(pk=object_id) + + +def display_mono(field): + def display(self, log): + content = getattr(log, field) + return monospace_format(escape(content)) + display.short_description = field + return display + + +def display_code(field): + def display(self, log): + return code_format(getattr(log, field)) + display.short_description = field + return display diff --git a/orchestra/api/__init__.py b/orchestra/api/__init__.py new file mode 100644 index 00000000..9f59e74c --- /dev/null +++ b/orchestra/api/__init__.py @@ -0,0 +1,2 @@ +from .options import * +from .actions import * diff --git a/orchestra/api/actions.py b/orchestra/api/actions.py new file mode 100644 index 00000000..d1b90264 --- /dev/null +++ b/orchestra/api/actions.py @@ -0,0 +1,30 @@ +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.response import Response + +from .serializers import SetPasswordSerializer + + +class SetPasswordApiMixin(object): + @action(detail=True, methods=['post'], serializer_class=SetPasswordSerializer) + def set_password(self, request, pk): + obj = self.get_object() + data = request.data + if isinstance(data, str): + data = { + 'password': data + } + serializer = SetPasswordSerializer(data=data) + if serializer.is_valid(): + obj.set_password(serializer.data['password']) + try: + obj.save(update_fields=['password']) + except ValueError: + # Some services don't store the password on the database + # update_fields=[] doesn't trigger post save! + obj.save() + return Response({ + 'status': 'password changed' + }) + else: + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) diff --git a/orchestra/api/helpers.py b/orchestra/api/helpers.py new file mode 100644 index 00000000..53803d9a --- /dev/null +++ b/orchestra/api/helpers.py @@ -0,0 +1,45 @@ +from django.urls import NoReverseMatch +from rest_framework.reverse import reverse + + +def link_wrap(view, view_names): + def wrapper(self, request, *args, **kwargs): + """ wrapper function that inserts HTTP links on view """ + links = [] + for name in view_names: + try: + url = reverse(name, request=self.request) + except NoReverseMatch: + url = reverse(name, args, kwargs, request=request) + links.append('<%s>; rel="%s"' % (url, name)) + response = view(self, request, *args, **kwargs) + response['Link'] = ', '.join(links) + return response + for attr in dir(view): + try: + setattr(wrapper, attr, getattr(view, attr)) + except: + pass + return wrapper + + +def insert_links(viewset, basename): + collection_links = ['api-root', '%s-list' % basename] + object_links = ['api-root', '%s-list' % basename, '%s-detail' % basename] + exception_links = ['api-root'] + list_links = ['api-root'] + retrieve_links = ['api-root', '%s-list' % basename] + # Determine any `@action` or `@link` decorated methods on the viewset + for methodname in dir(viewset): + method = getattr(viewset, methodname) + view_name = '%s-%s' % (basename, methodname.replace('_', '-')) + if hasattr(method, 'collection_bind_to_methods'): + list_links.append(view_name) + retrieve_links.append(view_name) + setattr(viewset, methodname, link_wrap(method, collection_links)) + elif hasattr(method, 'bind_to_methods'): + retrieve_links.append(view_name) + setattr(viewset, methodname, link_wrap(method, object_links)) + viewset.handle_exception = link_wrap(viewset.handle_exception, exception_links) + viewset.list = link_wrap(viewset.list, list_links) + viewset.retrieve = link_wrap(viewset.retrieve, retrieve_links) diff --git a/orchestra/api/options.py b/orchestra/api/options.py new file mode 100644 index 00000000..9e2a913e --- /dev/null +++ b/orchestra/api/options.py @@ -0,0 +1,94 @@ +from django.contrib.admin.options import get_content_type_for_model +from django.conf import settings as django_settings +from django.utils.encoding import force_str +from django.utils.module_loading import autodiscover_modules +from django.utils.translation import gettext as _ +from rest_framework.routers import DefaultRouter + +from orchestra import settings +from orchestra.utils.python import import_class + +from .helpers import insert_links + + +class LogApiMixin(object): + def create(self, request, *args, **kwargs): + from django.contrib.admin.models import ADDITION + response = super(LogApiMixin, self).create(request, *args, **kwargs) + message = _('Added.') + self.log(request, message, ADDITION, instance=self.serializer.instance) + return response + + def perform_create(self, serializer): + """ stores serializer for accessing instance on create() """ + super(LogApiMixin, self).perform_create(serializer) + self.serializer = serializer + + def update(self, request, *args, **kwargs): + from django.contrib.admin.models import CHANGE + response = super(LogApiMixin, self).update(request, *args, **kwargs) + message = _('Changed data') + self.log(request, message, CHANGE) + return response + + def partial_update(self, request, *args, **kwargs): + from django.contrib.admin.models import CHANGE + response = super(LogApiMixin, self).partial_update(request, *args, **kwargs) + message = _('Changed %s') % response.data + self.log(request, message, CHANGE) + return response + + def destroy(self, request, *args, **kwargs): + from django.contrib.admin.models import DELETION + message = _('Deleted') + self.log(request, message, DELETION) + response = super(LogApiMixin, self).destroy(request, *args, **kwargs) + return response + + def log(self, request, message, action, instance=None): + from django.contrib.admin.models import LogEntry + instance = instance or self.get_object() + LogEntry.objects.log_action( + user_id=request.user.pk, + content_type_id=get_content_type_for_model(instance).pk, + object_id=instance.pk, + object_repr=force_str(instance), + action_flag=action, + change_message=message, + ) + + +class LinkHeaderRouter(DefaultRouter): + def get_api_root_view(self, api_urls=None): + """ returns the root view, with all the linked collections """ + APIRoot = import_class(settings.ORCHESTRA_API_ROOT_VIEW) + APIRoot.router = self + return APIRoot.as_view() + + def register(self, prefix, viewset, basename=None): + """ inserts link headers on every viewset """ + if basename is None: + basename = self.get_default_basename(viewset) + insert_links(viewset, basename) + self.registry.append((prefix, viewset, basename)) + + def get_viewset(self, prefix_or_model): + for _prefix, viewset, __ in self.registry: + if _prefix == prefix_or_model or viewset.queryset.model == prefix_or_model: + return viewset + msg = "%s does not have a regiestered viewset" % prefix_or_model + raise KeyError(msg) + + def insert(self, prefix_or_model, name, field, **kwargs): + """ Dynamically add new fields to an existing serializer """ + viewset = self.get_viewset(prefix_or_model) + if viewset.serializer_class is None: + viewset.serializer_class = viewset().get_serializer_class() + viewset.serializer_class._declared_fields.update({name: field(**kwargs)}) + viewset.serializer_class.Meta.fields += (name,) + + +# Create a router and register our viewsets with it. +router = LinkHeaderRouter(trailing_slash=django_settings.APPEND_SLASH) + +autodiscover = lambda: (autodiscover_modules('api'), autodiscover_modules('serializers')) diff --git a/orchestra/api/root.py b/orchestra/api/root.py new file mode 100644 index 00000000..51fa23c1 --- /dev/null +++ b/orchestra/api/root.py @@ -0,0 +1,70 @@ +from rest_framework import views +from rest_framework.response import Response +from rest_framework.reverse import reverse + +from .. import settings +from ..core import services, accounts + + +class APIRoot(views.APIView): + names = ( + 'ORCHESTRA_SITE_NAME', + 'ORCHESTRA_SITE_VERBOSE_NAME' + ) + + def get(self, request, format=None): + root_url = reverse('api-root', request=request, format=format) + token_url = reverse('api-token-auth', request=request, format=format) + links = [ + '<%s>; rel="%s"' % (root_url, 'api-root'), + '<%s>; rel="%s"' % (token_url, 'api-get-auth-token'), + ] + body = { + 'accountancy': {}, + 'services': {}, + } + if not request.user.is_anonymous: + list_name = '{basename}-list' + detail_name = '{basename}-detail' + for prefix, viewset, basename in self.router.registry: + singleton_pk = getattr(viewset, 'singleton_pk', False) + if singleton_pk: + url_name = detail_name.format(basename=basename) + kwargs = { + 'pk': singleton_pk(viewset(), request) + } + else: + url_name = list_name.format(basename=basename) + kwargs = {} + url = reverse(url_name, request=request, format=format, kwargs=kwargs) + links.append('<%s>; rel="%s"' % (url, url_name)) + model = viewset.queryset.model + group = None + if model in services: + group = 'services' + menu = services[model].menu + if model in accounts: + group = 'accountancy' + menu = accounts[model].menu + if group and menu: + body[group][basename] = { + 'url': url, + 'verbose_name': model._meta.verbose_name, + 'verbose_name_plural': model._meta.verbose_name_plural, + } + headers = { + 'Link': ', '.join(links) + } + body.update({ + name.lower(): getattr(settings, name, None) + for name in self.names + }) + return Response(body, headers=headers) + + def options(self, request): + metadata = super(APIRoot, self).options(request) + metadata.data['settings'] = { + name.lower(): getattr(settings, name, None) + for name in self.names + } + return metadata diff --git a/orchestra/api/serializers.py b/orchestra/api/serializers.py new file mode 100644 index 00000000..4b6f4667 --- /dev/null +++ b/orchestra/api/serializers.py @@ -0,0 +1,114 @@ +import copy + +from django.core.exceptions import ValidationError +from django.db import models +from django.forms import widgets +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers +from rest_framework.utils import model_meta + +from ..core.validators import validate_password + + +class SetPasswordSerializer(serializers.Serializer): + password = serializers.CharField(max_length=128, label=_('Password'), + style={'widget': widgets.PasswordInput}, validators=[validate_password]) + + +class HyperlinkedModelSerializer(serializers.HyperlinkedModelSerializer): + """ support for postonly_fields, fields whose value can only be set on post """ + + def validate(self, attrs): + """ calls model.clean() """ + attrs = super(HyperlinkedModelSerializer, self).validate(attrs) + if isinstance(attrs, models.Model): + return attrs + validated_data = dict(attrs) + ModelClass = self.Meta.model + # Remove many-to-many relationships from validated_data. + info = model_meta.get_field_info(ModelClass) + for field_name, relation_info in info.relations.items(): + if relation_info.to_many and (field_name in validated_data): + validated_data.pop(field_name) + if self.instance: + # on update: Merge provided fields with instance field + instance = copy.deepcopy(self.instance) + for key, value in validated_data.items(): + setattr(instance, key, value) + else: + instance = ModelClass(**validated_data) + instance.clean() + return attrs + + def post_only_cleanning(self, instance, validated_data): + """ removes postonly_fields from attrs """ + model_attrs = dict(**validated_data) + post_only_fields = getattr(self, 'post_only_fields', None) + if instance is not None and post_only_fields: + for attr, value in validated_data.items(): + if attr in post_only_fields: + model_attrs.pop(attr) + return model_attrs + + def update(self, instance, validated_data): + """ removes postonly_fields from attrs when not posting """ + model_attrs = self.post_only_cleanning(instance, validated_data) + return super(HyperlinkedModelSerializer, self).update(instance, model_attrs) + + def partial_update(self, instance, validated_data): + """ removes postonly_fields from attrs when not posting """ + model_attrs = self.post_only_cleanning(instance, validated_data) + return super(HyperlinkedModelSerializer, self).partial_update(instance, model_attrs) + + +class RelatedHyperlinkedModelSerializer(HyperlinkedModelSerializer): + """ returns object on to_internal_value based on URL """ + def to_internal_value(self, data): + try: + url = data.get('url') + except AttributeError: + url = None + if not url: + raise ValidationError({ + 'url': "URL is required." + }) + account = self.get_account() + queryset = self.Meta.model.objects.filter(account=account) + self.fields['url'].queryset = queryset + obj = self.fields['url'].to_internal_value(url) + return obj + + +class SetPasswordHyperlinkedSerializer(HyperlinkedModelSerializer): + password = serializers.CharField(max_length=128, label=_('Password'), + validators=[validate_password], write_only=True, required=False, + style={'widget': widgets.PasswordInput}) + + def validate_password(self, value): + """ POST only password """ + if self.instance: + if value: + raise serializers.ValidationError(_("Can not set password")) + elif not value: + raise serializers.ValidationError(_("Password required")) + return value + + def validate(self, attrs): + """ remove password in case is not a real model field """ + try: + self.Meta.model._meta.get_field('password') + except models.FieldDoesNotExist: + pass + else: + password = attrs.pop('password', None) + attrs = super().validate(attrs) + if password is not None: + attrs['password'] = password + return attrs + + def create(self, validated_data): + password = validated_data.pop('password') + instance = self.Meta.model(**validated_data) + instance.set_password(password) + instance.save() + return instance diff --git a/orchestra/apps.py b/orchestra/apps.py new file mode 100644 index 00000000..dcf13f6f --- /dev/null +++ b/orchestra/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class OrchestraConfig(AppConfig): + name = 'orchestra' + verbose_name = 'Orchestra' diff --git a/orchestra/bin/celerybeat b/orchestra/bin/celerybeat new file mode 100755 index 00000000..00e8b35b --- /dev/null +++ b/orchestra/bin/celerybeat @@ -0,0 +1,285 @@ +#!/bin/bash +# ========================================================= +# celerybeat - Starts the Celery periodic task scheduler. +# ========================================================= +# +# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd +# +# See http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html#generic-init-scripts + +### BEGIN INIT INFO +# Provides: celerybeat +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery periodic task scheduler +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e +VERSION=10.0 +echo "celery init v${VERSION}." + +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unpriviliged users must use 'celery beat --detach'" + exit 1 +fi + + +# May be a runlevel symlink (e.g. S02celeryd) +if [ -L "$0" ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +# /etc/init.d/celerybeat: start and stop the celery periodic task scheduler daemon. + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully and make sure it has not been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully and make sure it has not been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully and make sure it has not been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +scripts="" + +if test -f /etc/default/celeryd; then + scripts="/etc/default/celeryd" + _config_sanity /etc/default/celeryd + . /etc/default/celeryd +fi + +EXTRA_CONFIG="/etc/default/${SCRIPT_NAME}" +if test -f "$EXTRA_CONFIG"; then + scripts="$scripts, $EXTRA_CONFIG" + _config_sanity "$EXTRA_CONFIG" + . "$EXTRA_CONFIG" +fi + +echo "Using configuration: $scripts" + +CELERY_BIN=${CELERY_BIN:-"celery"} +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/beat.pid" +DEFAULT_LOG_FILE="/var/log/celery/beat.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_CELERYBEAT="$CELERY_BIN beat" + +CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT} +CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +CELERYBEAT_USER=${CELERYBEAT_USER:-${CELERYD_USER:-$DEFAULT_USER}} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYBEAT_PID_FILE" ]; then + CELERYBEAT_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYBEAT_LOG_FILE" ]; then + CELERYBEAT_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +export CELERY_LOADER + +CELERYBEAT_OPTS="$CELERYBEAT_OPTS -f $CELERYBEAT_LOG_FILE -l $CELERYBEAT_LOG_LEVEL" + +if [ -n "$2" ]; then + CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2" +fi + +CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE` +CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE` + +# Extra start-stop-daemon options, like user/group. + +CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYBEAT_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYBEAT_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $*" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYBEAT_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYBEAT_USER'" + chown "$CELERYBEAT_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYBEAT_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYBEAT_GROUP'" + chgrp "$CELERYBEAT_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYBEAT_PID_DIR" + fi +} + + +create_paths () { + create_default_dir "$CELERYBEAT_LOG_DIR" + create_default_dir "$CELERYBEAT_PID_DIR" +} + + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + kill -0 $pid 1>/dev/null 2>&1 + if [ $? -eq 1 ]; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_beat () { + echo -n "Stopping ${SCRIPT_NAME}... " + if [ -f "$CELERYBEAT_PID_FILE" ]; then + wait_pid $(cat "$CELERYBEAT_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +_chuid () { + su "$CELERYBEAT_USER" -c "$CELERYBEAT $*" +} + +start_beat () { + echo "Starting ${SCRIPT_NAME}..." + _chuid $CELERY_APP_ARG $CELERYBEAT_OPTS $DAEMON_OPTS --detach \ + --pidfile="$CELERYBEAT_PID_FILE" +} + + + +case "$1" in + start) + check_dev_null + check_paths + start_beat + ;; + stop) + check_paths + stop_beat + ;; + reload|force-reload) + echo "Use start+stop" + ;; + restart) + echo "Restarting celery periodic task scheduler" + check_paths + stop_beat + check_dev_null + start_beat + ;; + create-paths) + check_dev_null + create_paths + ;; + check-paths) + check_dev_null + check_paths + ;; + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|create-paths}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 diff --git a/orchestra/bin/celeryd b/orchestra/bin/celeryd new file mode 100755 index 00000000..df918bca --- /dev/null +++ b/orchestra/bin/celeryd @@ -0,0 +1,387 @@ +#!/bin/sh -e +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# :Configuration file: /etc/default/celeryd +# +# See http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html#generic-init-scripts + + +### BEGIN INIT INFO +# Provides: celeryd +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery task worker daemon +### END INIT INFO +# +# +# To implement separate init scripts, copy this script and give it a different +# name: +# I.e., if my new application, "little-worker" needs an init, I +# should just use: +# +# cp /etc/init.d/celeryd /etc/init.d/little-worker +# +# You can then configure this by manipulating /etc/default/little-worker. +# +VERSION=10.0 +echo "celery init v${VERSION}." +if [ $(id -u) -ne 0 ]; then + echo "Error: This program can only be used by the root user." + echo " Unprivileged users must use the 'celery multi' utility, " + echo " or 'celery worker --detach'." + exit 1 +fi + + +# Can be a runlevel symlink (e.g. S02celeryd) +if [ -L "$0" ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" + +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/%n.pid" +DEFAULT_LOG_FILE="/var/log/celery/%n%I.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_NODES="celery" +DEFAULT_CELERYD="-m celery worker --detach" + +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/${SCRIPT_NAME}"} + +# Make sure executable configuration script is owned by root +_config_sanity() { + local path="$1" + local owner=$(ls -ld "$path" | awk '{print $3}') + local iwgrp=$(ls -ld "$path" | cut -b 6) + local iwoth=$(ls -ld "$path" | cut -b 9) + + if [ "$(id -u $owner)" != "0" ]; then + echo "Error: Config script '$path' must be owned by root!" + echo + echo "Resolution:" + echo "Review the file carefully and make sure it has not been " + echo "modified with mailicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change ownership of the script:" + echo " $ sudo chown root '$path'" + exit 1 + fi + + if [ "$iwoth" != "-" ]; then # S_IWOTH + echo "Error: Config script '$path' cannot be writable by others!" + echo + echo "Resolution:" + echo "Review the file carefully and make sure it has not been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi + if [ "$iwgrp" != "-" ]; then # S_IWGRP + echo "Error: Config script '$path' cannot be writable by group!" + echo + echo "Resolution:" + echo "Review the file carefully and make sure it has not been " + echo "modified with malicious intent. When sure the " + echo "script is safe to execute with superuser privileges " + echo "you can change the scripts permissions:" + echo " $ sudo chmod 640 '$path'" + exit 1 + fi +} + +if [ -f "$CELERY_DEFAULTS" ]; then + _config_sanity "$CELERY_DEFAULTS" + echo "Using config script: $CELERY_DEFAULTS" + . "$CELERY_DEFAULTS" +fi + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYD_PID_FILE" ]; then + CELERYD_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYD_LOG_FILE" ]; then + CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 +fi + +CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} +CELERY_BIN=${CELERY_BIN:-"celery"} +CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} +CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} + +export CELERY_LOADER + +if [ -n "$2" ]; then + CELERYD_OPTS="$CELERYD_OPTS $2" +fi + +CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` +CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYD_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" +fi + + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 75 # EX_TEMPFAIL + fi +} + + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $* (errno $?)" + exit 77 # EX_NOPERM + fi +} + +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYD_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYD_USER'" + chown "$CELERYD_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYD_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYD_GROUP'" + chgrp "$CELERYD_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYD_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYD_PID_DIR" + fi +} + +create_paths() { + create_default_dir "$CELERYD_LOG_DIR" + create_default_dir "$CELERYD_PID_DIR" +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + + +_get_pids() { + found_pids=0 + my_exitcode=0 + + for pid_file in "$CELERYD_PID_DIR"/*.pid; do + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pid_file)" + one_failed=true + my_exitcode=1 + else + found_pids=1 + echo "$pid" + fi + + if [ $found_pids -eq 0 ]; then + echo "${SCRIPT_NAME}: All nodes down" + exit $my_exitcode + fi + done +} + + +_chuid () { + su "$CELERYD_USER" -c "$CELERYD_MULTI $*" +} + + +start_workers () { + if [ ! -z "$CELERYD_ULIMIT" ]; then + ulimit $CELERYD_ULIMIT + fi + _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +dryrun () { + (C_FAKEFORK=1 start_workers --verbose) +} + + +stop_workers () { + _chuid stopwait $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers () { + _chuid restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +kill_workers() { + _chuid kill $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers_graceful () { + local worker_pids= + worker_pids=`_get_pids` + [ "$one_failed" ] && exit 1 + + for worker_pid in $worker_pids; do + local failed= + kill -HUP $worker_pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" + one_failed=true + else + echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +check_status () { + my_exitcode=0 + found_pids=0 + + local one_failed= + for pid_file in "$CELERYD_PID_DIR"/*.pid; do + if [ ! -r $pid_file ]; then + echo "${SCRIPT_NAME} is stopped: no pids were found" + one_failed=true + break + fi + + local node=`basename "$pid_file" .pid` + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pid_file)" + one_failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (node $node) (pid $pid) is stopped, but pid file exists!" + one_failed=true + else + echo "${SCRIPT_NAME} (node $node) (pid $pid) is running..." + fi + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +case "$1" in + start) + check_dev_null + check_paths + start_workers + ;; + + stop) + check_dev_null + check_paths + stop_workers + ;; + + reload|force-reload) + echo "Use restart" + ;; + + status) + check_status + ;; + + restart) + check_dev_null + check_paths + restart_workers + ;; + + graceful) + check_dev_null + restart_workers_graceful + ;; + + kill) + check_dev_null + kill_workers + ;; + + dryrun) + check_dev_null + dryrun + ;; + + try-restart) + check_dev_null + check_paths + restart_workers + ;; + + create-paths) + check_dev_null + create_paths + ;; + + check-paths) + check_dev_null + check_paths + ;; + + *) + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" + exit 64 # EX_USAGE + ;; +esac + +exit 0 diff --git a/orchestra/bin/celeryevcam b/orchestra/bin/celeryevcam new file mode 100755 index 00000000..623e1adb --- /dev/null +++ b/orchestra/bin/celeryevcam @@ -0,0 +1,226 @@ +#!/bin/bash +# ============================================ +# celeryd - Starts the Celery worker daemon. +# ============================================ +# +# :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} +# +# :Configuration file: /etc/default/celeryev | /etc/default/celeryd +# +# To configure celeryd you probably need to tell it where to chdir. +# +# EXAMPLE CONFIGURATION +# ===================== +# +# this is an example configuration for a Python project: +# +# /etc/default/celeryd: +# +# # Where to chdir at start. +# CELERYD_CHDIR="/opt/Myproject/" +# +# # Extra arguments to celeryev +# CELERYEV_OPTS="-x" +# +# # Name of the celery config module.# +# CELERY_CONFIG_MODULE="celeryconfig" +# +# # Camera class to use (required) +# CELERYEV_CAM = "myapp.Camera" +# +# EXAMPLE DJANGO CONFIGURATION +# ============================ +# +# # Where the Django project is. +# CELERYD_CHDIR="/opt/Project/" +# +# # Name of the projects settings module. +# export DJANGO_SETTINGS_MODULE="MyProject.settings" +# +# # Path to celeryd +# CELERYEV="/opt/Project/manage.py" +# +# # Extra arguments to manage.py +# CELERYEV_OPTS="celeryev" +# +# # Camera class to use (required) +# CELERYEV_CAM="djcelery.snapshot.Camera" +# +# AVAILABLE OPTIONS +# ================= +# +# * CELERYEV_OPTS +# Additional arguments to celeryd, see `celeryd --help` for a list. +# +# * CELERYD_CHDIR +# Path to chdir at start. Default is to stay in the current directory. +# +# * CELERYEV_PID_FILE +# Full path to the pidfile. Default is /var/run/celeryd.pid. +# +# * CELERYEV_LOG_FILE +# Full path to the celeryd logfile. Default is /var/log/celeryd.log +# +# * CELERYEV_LOG_LEVEL +# Log level to use for celeryd. Default is INFO. +# +# * CELERYEV +# Path to the celeryev program. Default is `celeryev`. +# You can point this to an virtualenv, or even use manage.py for django. +# +# * CELERYEV_USER +# User to run celeryev as. Default is current user. +# +# * CELERYEV_GROUP +# Group to run celeryev as. Default is current user. +# +# * VIRTUALENV +# Full path to the virtualenv environment to activate. Default is none. + +### BEGIN INIT INFO +# Provides: celeryev +# Required-Start: $network $local_fs $remote_fs postgresql rabbitmq-server +# Required-Stop: $network $local_fs $remote_fs postgresql rabbitmq-server +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: celery event snapshots +### END INIT INFO + +# Cannot use set -e/bash -e since the kill -0 command will abort +# abnormally in the absence of a valid process ID. +#set -e + +DEFAULT_PID_FILE="/var/run/celeryev.pid" +DEFAULT_LOG_FILE="/var/log/celeryev.log" +DEFAULT_LOG_LEVEL="INFO" +DEFAULT_CELERYEV="/usr/bin/celeryev" + +if test -f /etc/default/celeryd; then + . /etc/default/celeryd +fi + +if test -f /etc/default/celeryev; then + . /etc/default/celeryev +fi + +CELERYEV=${CELERYEV:-$DEFAULT_CELERYEV} +CELERYEV_PID_FILE=${CELERYEV_PID_FILE:-${CELERYEV_PIDFILE:-$DEFAULT_PID_FILE}} +CELERYEV_LOG_FILE=${CELERYEV_LOG_FILE:-${CELERYEV_LOGFILE:-$DEFAULT_LOG_FILE}} +CELERYEV_LOG_LEVEL=${CELERYEV_LOG_LEVEL:-${CELERYEV_LOG_LEVEL:-$DEFAULT_LOG_LEVEL}} + +export CELERY_LOADER + +if [ -z "$CELERYEV_CAM" ]; then + echo "Missing CELERYEV_CAM variable" 1>&2 + exit +fi + +CELERYEV_OPTS="$CELERYEV_OPTS -f $CELERYEV_LOG_FILE -l $CELERYEV_LOG_LEVEL -c $CELERYEV_CAM" + +if [ -n "$2" ]; then + CELERYEV_OPTS="$CELERYEV_OPTS $2" +fi + +CELERYEV_LOG_DIR=`dirname $CELERYEV_LOG_FILE` +CELERYEV_PID_DIR=`dirname $CELERYEV_PID_FILE` +if [ ! -d "$CELERYEV_LOG_DIR" ]; then + mkdir -p $CELERYEV_LOG_DIR +fi +if [ ! -d "$CELERYEV_PID_DIR" ]; then + mkdir -p $CELERYEV_PID_DIR +fi + +# Extra start-stop-daemon options, like user/group. +if [ -n "$CELERYEV_USER" ]; then + DAEMON_OPTS="$DAEMON_OPTS --uid $CELERYEV_USER" + chown "$CELERYEV_USER" $CELERYEV_LOG_DIR $CELERYEV_PID_DIR +fi +if [ -n "$CELERYEV_GROUP" ]; then + DAEMON_OPTS="$DAEMON_OPTS --gid $CELERYEV_GROUP" + chgrp "$CELERYEV_GROUP" $CELERYEV_LOG_DIR $CELERYEV_PID_DIR +fi + +CELERYEV_CHDIR=${CELERYEV_CHDIR:-$CELERYD_CHDIR} +if [ -n "$CELERYEV_CHDIR" ]; then + DAEMON_OPTS="$DAEMON_OPTS --workdir $CELERYEV_CHDIR" +fi + + +export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" + +check_dev_null() { + if [ ! -c /dev/null ]; then + echo "/dev/null is not a character device!" + exit 1 + fi +} + +wait_pid () { + pid=$1 + forever=1 + i=0 + while [ $forever -gt 0 ]; do + kill -0 $pid 1>/dev/null 2>&1 + if [ $? -eq 1 ]; then + echo "OK" + forever=0 + else + kill -TERM "$pid" + i=$((i + 1)) + if [ $i -gt 60 ]; then + echo "ERROR" + echo "Timed out while stopping (30s)" + forever=0 + else + sleep 0.5 + fi + fi + done +} + + +stop_evcam () { + echo -n "Stopping celeryev..." + if [ -f "$CELERYEV_PID_FILE" ]; then + wait_pid $(cat "$CELERYEV_PID_FILE") + else + echo "NOT RUNNING" + fi +} + +start_evcam () { + echo "Starting celeryev..." + if [ -n "$VIRTUALENV" ]; then + source $VIRTUALENV/bin/activate + fi + $CELERYEV $CELERYEV_OPTS $DAEMON_OPTS --detach \ + --pidfile="$CELERYEV_PID_FILE" +} + + + +case "$1" in + start) + check_dev_null + start_evcam + ;; + stop) + stop_evcam + ;; + reload|force-reload) + echo "Use start+stop" + ;; + restart) + echo "Restarting celery event snapshots" "celeryev" + stop_evcam + check_dev_null + start_evcam + ;; + + *) + echo "Usage: /etc/init.d/celeryev {start|stop|restart}" + exit 1 +esac + +exit 0 + diff --git a/orchestra/bin/django_bash_completion.sh b/orchestra/bin/django_bash_completion.sh new file mode 100755 index 00000000..8f852117 --- /dev/null +++ b/orchestra/bin/django_bash_completion.sh @@ -0,0 +1,72 @@ +# ######################################################################### +# This bash script adds tab-completion feature to django-admin.py and +# manage.py. +# +# Testing it out without installing +# ================================= +# +# To test out the completion without "installing" this, just run this file +# directly, like so: +# +# . ~/path/to/django_bash_completion +# +# Note: There's a dot ('.') at the beginning of that command. +# +# After you do that, tab completion will immediately be made available in your +# current Bash shell. But it won't be available next time you log in. +# +# Installing +# ========== +# +# To install this, point to this file from your .bash_profile, like so: +# +# . ~/path/to/django_bash_completion +# +# Do the same in your .bashrc if .bashrc doesn't invoke .bash_profile. +# +# Settings will take effect the next time you log in. +# +# Uninstalling +# ============ +# +# To uninstall, just remove the line from your .bash_profile and .bashrc. + +_django_completion() +{ + COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + DJANGO_AUTO_COMPLETE=1 $1 ) ) +} +complete -F _django_completion -o default django-admin.py manage.py django-admin + +_python_django_completion() +{ + if [[ ${COMP_CWORD} -ge 2 ]]; then + PYTHON_EXE=${COMP_WORDS[0]##*/} + echo $PYTHON_EXE | egrep "python([2-9]\.[0-9])?" >/dev/null 2>&1 + if [[ $? == 0 ]]; then + PYTHON_SCRIPT=${COMP_WORDS[1]##*/} + echo $PYTHON_SCRIPT | egrep "manage\.py|django-admin(\.py)?" >/dev/null 2>&1 + if [[ $? == 0 ]]; then + COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]:1}" \ + COMP_CWORD=$(( COMP_CWORD-1 )) \ + DJANGO_AUTO_COMPLETE=1 ${COMP_WORDS[*]} ) ) + fi + fi + fi +} + +# Support for multiple interpreters. +unset pythons +if command -v whereis &>/dev/null; then + python_interpreters=$(whereis python | cut -d " " -f 2-) + for python in $python_interpreters; do + pythons="${pythons} ${python##*/}" + done + pythons=$(echo $pythons | tr " " "\n" | sort -u | tr "\n" " ") +else + pythons=python +fi + +complete -F _python_django_completion -o default $pythons + diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin new file mode 100755 index 00000000..a5b6e1de --- /dev/null +++ b/orchestra/bin/orchestra-admin @@ -0,0 +1,246 @@ +#!/bin/bash + +set -u +set -e + +bold=$(tput -T ${TERM:-xterm} bold) +normal=$(tput -T ${TERM:-xterm} sgr0) + + +PYTHON_BIN='python3' + +function help () { + if [[ $# -gt 1 ]]; then + CMD="print_${2}_help" + $CMD + else + print_help + fi +} + + +function print_help () { + cat <<- EOF + + ${bold}NAME${normal} + ${bold}orchestra-admin${normal} - Orchetsra administration script + + ${bold}OPTIONS${normal} + ${bold}install_requirements${normal} + Installs Orchestra requirements using apt-get and pip + + ${bold}startproject${normal} + Creates a new Django-orchestra instance + + ${bold}help${normal} + Displays this help text or related help page as argument + for example: + ${bold}orchestra-admin help startproject${normal} + + EOF +} + + +show () { + echo " ${bold}\$ ${@}${normal}" +} +export -f show + + +run () { + show "${@}" + "${@}" +} +export -f run + + +check_root () { + [ $(whoami) != 'root' ] && { echo -e "\nErr. This should be run as root\n" >&2; exit 1; } +} +export -f check_root + + +get_orchestra_dir () { + if ! $(echo "import orchestra" | $PYTHON_BIN 2> /dev/null); then + echo -e "\norchestra not installed.\n" >&2 + exit 1 + fi + PATH=$(echo "import orchestra, os; print(os.path.dirname(os.path.realpath(orchestra.__file__)))" | $PYTHON_BIN) + echo $PATH +} +export -f get_orchestra_dir + + +function print_install_requirements_help () { + cat <<- EOF + + ${bold}NAME${normal} + ${bold}orchetsra-admin install_requirements${normal} - Installs all Orchestra requirements using apt-get and pip + + ${bold}OPTIONS${normal} + ${bold}-t, --testing${normal} + Install Orchestra normal requirements plus those needed for running functional tests + + ${bold}-h, --help${normal} + Displays this help text + + EOF +} + + +function install_requirements () { + opts=$(getopt -o h,t -l help,testing -- "$@") || exit 1 + set -- $opts + testing=false + + while [ $# -gt 0 ]; do + case $1 in + -h|--help) print_deploy_help; exit 0 ;; + -t|--testing) testing=true; shift ;; + (--) shift; break;; + (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift + done + unset OPTIND + unset opt + + check_root || true + ORCHESTRA_PATH=$(get_orchestra_dir) || true + + # Make sure locales are in place before installing postgres + if [[ $({ perl --help > /dev/null; } 2>&1|grep 'locale failed') ]]; then + run sed -i "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen + run locale-gen + update-locale LANG=en_US.UTF-8 + fi + + # lxml: libxml2-dev, libxslt1-dev, zlib1g-dev + APT="bind9utils \ + ca-certificates \ + gettext \ + libcrack2-dev \ + libxml2-dev \ + libxslt1-dev \ + python3 \ + python3-pip \ + python3-dev \ + ssh-client \ + wget \ + xvfb \ + zlib1g-dev" + if $testing; then + APT="${APT} \ + git \ + iceweasel \ + dnsutils" + fi + + run apt-get update + run apt-get install -y $APT + + # Install ca certificates before executing pip install + if [[ ! -e /usr/local/share/ca-certificates/cacert.org ]]; then + mkdir -p /usr/local/share/ca-certificates/cacert.org + wget -P /usr/local/share/ca-certificates/cacert.org \ + http://www.cacert.org/certs/root.crt \ + http://www.cacert.org/certs/class3.crt + update-ca-certificates + fi + + # cracklib and lxml are excluded on the requirements.txt because they need unconvinient system dependencies + PIP="$(wget http://git.io/orchestra-requirements.txt -O - | tr '\n' ' ') \ + cracklib \ + lxml==3.3.5" + if $testing; then + PIP="${PIP} \ + selenium \ + xvfbwrapper \ + freezegun==0.3.14 \ + coverage \ + flake8 \ + django-debug-toolbar==1.3.0 \ + django-nose==1.4.4 \ + sqlparse \ + pyinotify \ + PyMySQL" + fi + + run pip3 install $PIP + + # Install a more recent version of wkhtmltopdf (0.12.2) (PDF page number support) + wkhtmltox_version=$(dpkg --list | grep wkhtmltox | awk {'print $3'}) + minor=$(echo -e "$wkhtmltox_version\n0.12.2.1" | sort -V | head -n 1) + if [[ ! $wkhtmltox_version ]] || [[ $wkhtmltox_version != 0.12.2.1 && $minor == ${wkhtmltox_version} ]]; then + wkhtmltox=$(mktemp) + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_amd64.deb -O ${wkhtmltox} + dpkg -i ${wkhtmltox} || { echo "Installing missing dependencies for wkhtmltox..." && apt-get -f -y install; } + fi +} +export -f install_requirements + + +print_startproject_help () { + cat <<- EOF + + ${bold}NAME${normal} + ${bold}orchestra-admin startproject${normal} - Create a new Django-Orchestra instance + + ${bold}SYNOPSIS${normal} + Options: [ -h ] + + ${bold}OPTIONS${normal} + ${bold}-h, --help${normal} + This help message + + ${bold}EXAMPLES${normal} + orchestra-admin startproject controlpanel + + EOF +} + + +function startproject () { + local PROJECT_NAME="$2"; shift + + opts=$(getopt -o h -l help -- "$@") || exit 1 + set -- $opts + + set -- $opts + while [ $# -gt 0 ]; do + case $1 in + -h|--help) print_startproject_help; exit 0 ;; + (--) shift; break;; + (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift + done + + unset OPTIND + unset opt + + [ $(whoami) == 'root' ] && { echo -e "\nYou don't want to run this as root\n" >&2; exit 1; } + ORCHESTRA_PATH=$(get_orchestra_dir) || { echo "Error getting orchestra dir"; exit 1; } + if [[ ! -e $PROJECT_NAME/manage.py ]]; then + run django-admin.py startproject $PROJECT_NAME --template="${ORCHESTRA_PATH}/conf/project_template" + # This is a workaround for this issue https://github.com/pypa/pip/issues/317 + run chmod +x $PROJECT_NAME/manage.py + # End of workaround ### + else + echo "Not cloning: $PROJECT_NAME already exists." + fi + # Install bash autocompletition for django commands + if [[ ! $(grep 'source $HOME/.django_bash_completion.sh' ~/.bashrc &> /dev/null) ]]; then + # run wget https://raw.github.com/django/django/master/extras/django_bash_completion \ + # --no-check-certificate -O ~/.django_bash_completion.sh + cp ${ORCHESTRA_PATH}/bin/django_bash_completion.sh ~/.django_bash_completion.sh + echo 'source $HOME/.django_bash_completion.sh' >> ~/.bashrc + fi +} +export -f startproject + + +[ $# -lt 1 ] && { print_help; exit 1; } +$1 "${@}" diff --git a/orchestra/bin/orchestra-beat b/orchestra/bin/orchestra-beat new file mode 100755 index 00000000..b11eda09 --- /dev/null +++ b/orchestra/bin/orchestra-beat @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +# High performance alternative to beat management command +# Looks for pending work before firing up all the Django machinery on separate processes +# +# Handles orchestra.contrib.tasks periodic_tasks and orchestra.contrib.mailer queued mails +# +# USAGE: beat /path/to/project/manage.py + + +import json +import os +import re +import sys +from datetime import datetime, timedelta + +from orchestra.utils.sys import run, join, LockFile + + +class crontab_parser(object): + """ + from celery.schedules import crontab_parser + Too expensive to import celery + """ + ParseException = ValueError + + _range = r'(\w+?)-(\w+)' + _steps = r'/(\w+)?' + _star = r'\*' + + def __init__(self, max_=60, min_=0): + self.max_ = max_ + self.min_ = min_ + self.pats = ( + (re.compile(self._range + self._steps), self._range_steps), + (re.compile(self._range), self._expand_range), + (re.compile(self._star + self._steps), self._star_steps), + (re.compile('^' + self._star + '$'), self._expand_star), + ) + + def parse(self, spec): + acc = set() + for part in spec.split(','): + if not part: + raise self.ParseException('empty part') + acc |= set(self._parse_part(part)) + return acc + + def _parse_part(self, part): + for regex, handler in self.pats: + m = regex.match(part) + if m: + return handler(m.groups()) + return self._expand_range((part, )) + + def _expand_range(self, toks): + fr = self._expand_number(toks[0]) + if len(toks) > 1: + to = self._expand_number(toks[1]) + if to < fr: # Wrap around max_ if necessary + return (list(range(fr, self.min_ + self.max_)) + + list(range(self.min_, to + 1))) + return list(range(fr, to + 1)) + return [fr] + + def _range_steps(self, toks): + if len(toks) != 3 or not toks[2]: + raise self.ParseException('empty filter') + return self._expand_range(toks[:2])[::int(toks[2])] + + def _star_steps(self, toks): + if not toks or not toks[0]: + raise self.ParseException('empty filter') + return self._expand_star()[::int(toks[0])] + def _expand_star(self, *args): + return list(range(self.min_, self.max_ + self.min_)) + + def _expand_number(self, s): + if isinstance(s, str) and s[0] == '-': + raise self.ParseException('negative numbers not supported') + try: + i = int(s) + except ValueError: + try: + i = weekday(s) + except KeyError: + raise ValueError('Invalid weekday literal {0!r}.'.format(s)) + max_val = self.min_ + self.max_ - 1 + if i > max_val: + raise ValueError( + 'Invalid end range: {0} > {1}.'.format(i, max_val)) + if i < self.min_: + raise ValueError( + 'Invalid beginning range: {0} < {1}.'.format(i, self.min_)) + return i + + +class Setting(object): + def __init__(self, manage): + self.manage = manage + self.settings_file = self.get_settings_file(manage) + + def get_settings(self): + """ get db settings from settings.py file without importing """ + settings = {'__file__': self.settings_file} + with open(self.settings_file) as f: + content = '' + for line in f.readlines(): + # This is very costly, skip + if not line.startswith(('import djcelery', 'djcelery.setup_loader()')): + content += line + exec(content, settings) + return settings + + def get_settings_file(self, manage): + with open(manage, 'r') as handler: + regex = re.compile(r'"DJANGO_SETTINGS_MODULE"\s*,\s*"([^"]+)"') + for line in handler.readlines(): + match = regex.search(line) + if match: + settings_module = match.groups()[0] + settings_file = os.path.join(*settings_module.split('.')) + '.py' + settings_file = os.path.join(os.path.dirname(manage), settings_file) + return settings_file + raise ValueError("settings module not found in %s" % manage) + + +class DB(object): + def __init__(self, settings): + self.settings = settings['DATABASES']['default'] + + def connect(self): + if self.settings['ENGINE'] == 'django.db.backends.sqlite3': + import sqlite3 + self.conn = sqlite3.connect(self.settings['NAME']) + elif self.settings['ENGINE'] == 'django.db.backends.postgresql_psycopg2': + import psycopg2 + self.conn = psycopg2.connect("dbname='{NAME}' user='{USER}' host='{HOST}' password='{PASSWORD}'".format(**self.settings)) + else: + raise ValueError("%s engine not supported." % self.settings['ENGINE']) + + def query(self, query): + cur = self.conn.cursor() + try: + cur.execute(query) + result = cur.fetchall() + finally: + cur.close() + return result + + def close(self): + self.conn.close() + + +def fire_pending_tasks(manage, db): + def get_tasks(db): + enabled = 1 if 'sqlite' in db.settings['ENGINE'] else True + query = ( + "SELECT c.minute, c.hour, c.day_of_week, c.day_of_month, c.month_of_year, p.id " + "FROM djcelery_periodictask as p, djcelery_crontabschedule as c " + "WHERE p.crontab_id = c.id AND p.enabled = {}" + ).format(enabled) + return db.query(query) + + def is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): + n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = now + return ( + n_minute in crontab_parser(60).parse(minute) and + n_hour in crontab_parser(24).parse(hour) and + n_day_of_week in crontab_parser(7).parse(day_of_week) and + n_day_of_month in crontab_parser(31, 1).parse(day_of_month) and + n_month_of_year in crontab_parser(12, 1).parse(month_of_year) + ) + + now = datetime.utcnow() + now = tuple(map(int, now.strftime("%M %H %w %d %m").split())) + for minute, hour, day_of_week, day_of_month, month_of_year, task_id in get_tasks(db): + if is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): + command = 'python3 -W ignore::DeprecationWarning {manage} runtask {task_id}'.format( + manage=manage, task_id=task_id) + proc = run(command, run_async=True) + yield proc + + +def fire_pending_messages(settings, db): + def has_pending_messages(settings, db): + MAILER_DEFERE_SECONDS = settings.get('MAILER_DEFERE_SECONDS', (300, 600, 60*60, 60*60*24)) + now = datetime.utcnow() + query_or = [] + + for num, seconds in enumerate(MAILER_DEFERE_SECONDS): + delta = timedelta(seconds=seconds) + epoch = now-delta + query_or.append("""(mailer_message.retries = %i AND mailer_message.last_try <= '%s')""" + % (num, epoch.isoformat().replace('T', ' '))) + query = """\ + SELECT 1 FROM mailer_message + WHERE (mailer_message.state = 'QUEUED' + OR (mailer_message.state = 'DEFERRED' AND (%s))) LIMIT 1""" % ' OR '.join(query_or) + return bool(db.query(query)) + + if has_pending_messages(settings, db): + command = 'python3 -W ignore::DeprecationWarning {manage} sendpendingmessages'.format(manage=manage) + proc = run(command, run_async=True) + yield proc + + +if __name__ == "__main__": + with LockFile('/dev/shm/beat.lock', expire=20): + manage = sys.argv[1] + procs = [] + settings = Setting(manage).get_settings() + db = DB(settings) + db.connect() + try: + # Non-blocking loop, we need to finish this in time for the next minute. + if 'orchestra.contrib.tasks' in settings['INSTALLED_APPS']: + if settings.get('TASKS_BACKEND', 'thread') in ('thread', 'process'): + for proc in fire_pending_tasks(manage, db): + procs.append(proc) + if 'orchestra.contrib.mailer' in settings['INSTALLED_APPS']: + for proc in fire_pending_messages(settings, db): + procs.append(proc) + finally: + db.close() + sys.exit(0) diff --git a/orchestra/bin/sieve-test b/orchestra/bin/sieve-test new file mode 100755 index 00000000..da753038 Binary files /dev/null and b/orchestra/bin/sieve-test differ diff --git a/orchestra/conf/__init__.py b/orchestra/conf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/conf/project_template/locale/.gitignore b/orchestra/conf/project_template/locale/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/conf/project_template/manage.py b/orchestra/conf/project_template/manage.py new file mode 100755 index 00000000..ee4b9652 --- /dev/null +++ b/orchestra/conf/project_template/manage.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import os +import sys + + +if __name__ == "__main__": + if sys.version_info < (3, 3): + cmd = ' '.join(sys.argv) + sys.stderr.write("Sorry, Orchestra requires at least Python 3.3, try with:\n$ python3 %s\n" % cmd) + sys.exit(1) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings") + from django.core.management import execute_from_command_line + execute_from_command_line(sys.argv) diff --git a/orchestra/conf/project_template/media/.gitignore b/orchestra/conf/project_template/media/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/conf/project_template/project_name/__init__.py b/orchestra/conf/project_template/project_name/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/conf/project_template/project_name/settings.py b/orchestra/conf/project_template/project_name/settings.py new file mode 100644 index 00000000..82b244f9 --- /dev/null +++ b/orchestra/conf/project_template/project_name/settings.py @@ -0,0 +1,275 @@ +""" +Django settings for {{ project_name }} project. + +Generated by 'django-admin startproject' using Django {{ django_version }}. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '{{ secret_key }}' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + +# Application definition + +INSTALLED_APPS = [ + # django-orchestra apps + 'orchestra', + 'orchestra.contrib.accounts', + 'orchestra.contrib.systemusers', + 'orchestra.contrib.contacts', + 'orchestra.contrib.orchestration', + 'orchestra.contrib.bills', + 'orchestra.contrib.payments', + 'orchestra.contrib.tasks', + 'orchestra.contrib.mailer', + 'orchestra.contrib.history', + 'orchestra.contrib.issues', + 'orchestra.contrib.services', + 'orchestra.contrib.plans', + 'orchestra.contrib.orders', + 'orchestra.contrib.domains', + 'orchestra.contrib.mailboxes', + 'orchestra.contrib.lists', + 'orchestra.contrib.webapps', + 'orchestra.contrib.websites', + 'orchestra.contrib.letsencrypt', + 'orchestra.contrib.databases', + 'orchestra.contrib.vps', + 'orchestra.contrib.saas', + 'orchestra.contrib.miscellaneous', + + # Third-party apps + 'django_extensions', + 'djcelery', + 'fluent_dashboard', + 'admin_tools', + 'admin_tools.theming', + 'admin_tools.menu', + 'admin_tools.dashboard', + 'rest_framework', + 'rest_framework.authtoken', + 'django_filters', + 'passlib.ext.django', + 'django_countries', +# 'debug_toolbar', + + # Django.contrib + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.admin.apps.SimpleAdminConfig', + + # Last to load + 'orchestra.contrib.resources', + 'orchestra.contrib.settings', +# 'django_nose', +] + + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + + 'orchestra.core.caches.RequestCacheMiddleware', + # also handles transations, ATOMIC_REQUESTS does not wrap middlewares + 'orchestra.contrib.orchestration.middlewares.OperationsMiddleware', +] + + +ROOT_URLCONF = '{{ project_name }}.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'orchestra.core.context_processors.site', + ], + 'loaders': [ + 'admin_tools.template_loaders.Loader', + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ], + }, + }, +] + + +WSGI_APPLICATION = '{{ project_name }}.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'USER': '', # Not used with sqlite3. + 'PASSWORD': '', # Not used with sqlite3. + 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '', # Set to empty string for default. Not used with sqlite3. + 'CONN_MAX_AGE': 60*10 # Enable persistent connections + } +} + + +# Password validation +# https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/{{ docs_version }}/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + + +try: + TIME_ZONE = open('/etc/timezone', 'r').read().strip() +except IOError: + TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/{{ docs_version }}/howto/static-files/ + +STATIC_URL = '/static/' + + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/home/media/media.lawrence.com/static/" +STATIC_ROOT = os.path.join(BASE_DIR, 'static') + +# Absolute filesystem path to the directory that will hold user-uploaded files. +MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + + +# Path used for database translations files +LOCALE_PATHS = ( + os.path.join(BASE_DIR, 'locale'), +) + +ORCHESTRA_SITE_NAME = '{{ project_name }}' + + +AUTH_USER_MODEL = 'accounts.Account' + + +AUTHENTICATION_BACKENDS = [ + 'orchestra.permissions.auth.OrchestraPermissionBackend', + 'django.contrib.auth.backends.ModelBackend', +] + + +EMAIL_BACKEND = 'orchestra.contrib.mailer.backends.EmailBackend' + + +# Needed for Bulk operations +DATA_UPLOAD_MAX_NUMBER_FIELDS = None + + +################################# +## 3RD PARTY APPS CONIGURATION ## +################################# + +# Admin Tools +ADMIN_TOOLS_MENU = 'orchestra.admin.menu.OrchestraMenu' + +# Fluent dashboard +ADMIN_TOOLS_INDEX_DASHBOARD = 'orchestra.admin.dashboard.OrchestraIndexDashboard' +FLUENT_DASHBOARD_ICON_THEME = '../orchestra/icons' + + +# Django-celery +import djcelery +djcelery.setup_loader() +CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' + + +# rest_framework +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': ( + 'orchestra.permissions.api.OrchestraPermissionBackend', + ), + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.TokenAuthentication', + ), + 'DEFAULT_FILTER_BACKENDS': ( + ('django_filters.rest_framework.DjangoFilterBackend',) + ), +} + + +# Use a UNIX compatible hash +PASSLIB_CONFIG = ( + "[passlib]\n" + "schemes = sha512_crypt, django_pbkdf2_sha256, django_pbkdf2_sha1, " + " django_bcrypt, django_bcrypt_sha256, django_salted_sha1, des_crypt, " + " django_salted_md5, django_des_crypt, hex_md5, bcrypt, phpass\n" + "default = sha512_crypt\n" + "deprecated = django_pbkdf2_sha1, django_salted_sha1, django_salted_md5, " + " django_des_crypt, des_crypt, hex_md5\n" + "django_pbkdf2_sha256__min_rounds = 10000\n" + "sha512_crypt__min_rounds = 80000\n" + "staff__django_pbkdf2_sha256__default_rounds = 12500\n" + "staff__sha512_crypt__default_rounds = 100000\n" + "superuser__django_pbkdf2_sha256__default_rounds = 15000\n" + "superuser__sha512_crypt__default_rounds = 120000\n" +) + + +SHELL_PLUS_PRE_IMPORTS = ( + ('orchestra.contrib.orchestration.managers', ('orchestrate',)), +) diff --git a/orchestra/conf/project_template/project_name/urls.py b/orchestra/conf/project_template/project_name/urls.py new file mode 100644 index 00000000..3ae27421 --- /dev/null +++ b/orchestra/conf/project_template/project_name/urls.py @@ -0,0 +1,6 @@ +from django.conf.urls import include, url + + +urlpatterns = [ + url(r'', include('orchestra.urls')), +] diff --git a/orchestra/conf/project_template/project_name/wsgi.py b/orchestra/conf/project_template/project_name/wsgi.py new file mode 100644 index 00000000..94d60c8c --- /dev/null +++ b/orchestra/conf/project_template/project_name/wsgi.py @@ -0,0 +1,14 @@ +""" +WSGI config for {{ project_name }} project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ +""" + +import os +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings") + +from django.core.wsgi import get_wsgi_application +application = get_wsgi_application() diff --git a/orchestra/contrib/__init__.py b/orchestra/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/accounts/__init__.py b/orchestra/contrib/accounts/__init__.py new file mode 100644 index 00000000..122f30b7 --- /dev/null +++ b/orchestra/contrib/accounts/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.accounts.apps.AccountConfig' diff --git a/orchestra/contrib/accounts/actions.py b/orchestra/contrib/accounts/actions.py new file mode 100644 index 00000000..0bf45a00 --- /dev/null +++ b/orchestra/contrib/accounts/actions.py @@ -0,0 +1,287 @@ +from functools import partial, wraps + +from django.contrib import messages +from django.contrib.admin import helpers +from django.contrib.admin.utils import NestedObjects, quote +from django.contrib.auth import get_permission_codename +from django.urls import reverse, NoReverseMatch +from django.db import router +from django.shortcuts import redirect, render +from django.template.response import TemplateResponse +from django.utils import timezone +from django.utils.encoding import force_str +from django.utils.html import format_html +from django.utils.text import capfirst +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.core import services + +from . import settings + + +def list_contacts(modeladmin, request, queryset): + ids = queryset.order_by().values_list('id', flat=True).distinct() + if not ids: + messages.warning(request, "Select at least one account.") + return + url = reverse('admin:contacts_contact_changelist') + url += '?account__in=%s' % ','.join(map(str, ids)) + return redirect(url) +list_contacts.short_description = _("List contacts") + + +def list_accounts(modeladmin, request, queryset): + accounts = queryset.order_by().values_list('account_id', flat=True).distinct() + if not accounts: + messages.warning(request, "Select at least one instance.") + return + url = reverse('admin:contacts_contact_changelist') + url += '?id__in=%s' % ','.join(map(str, accounts)) + return redirect(url) +list_accounts.short_description = _("List accounts") + + +def service_report(modeladmin, request, queryset): + # TODO resources + accounts = [] + fields = [] + registered_services = services.get() + # First we get related manager names to fire a prefetch related + for name, field in queryset.model._meta.fields_map.items(): + model = field.related_model + if model in registered_services and model != queryset.model: + fields.append((model, name)) + fields = sorted(fields, key=lambda f: f[0]._meta.verbose_name_plural.lower()) + fields = [field for model, field in fields] + + for account in queryset.prefetch_related(*fields): + items = [] + for field in fields: + related_manager = getattr(account, field) + items.append((related_manager.model._meta, related_manager.all())) + accounts.append((account, items)) + + context = { + 'accounts': accounts, + 'date': timezone.now().today() + } + return render(request, settings.ACCOUNTS_SERVICE_REPORT_TEMPLATE, context) + + +def delete_related_services(modeladmin, request, queryset): + opts = modeladmin.model._meta + app_label = opts.app_label + + using = router.db_for_write(modeladmin.model) + collector = NestedObjects(using=using) + collector.collect(queryset) + registered_services = services.get() + related_services = [] + to_delete = [] + + admin_site = modeladmin.admin_site + + def format(obj, account=False): + has_admin = obj.__class__ in admin_site._registry + opts = obj._meta + no_edit_link = '%s: %s' % (capfirst(opts.verbose_name), force_str(obj)) + + if has_admin: + try: + admin_url = reverse( + 'admin:%s_%s_change' % (opts.app_label, opts.model_name), + None, (quote(obj._get_pk_val()),) + ) + except NoReverseMatch: + # Change url doesn't exist -- don't display link to edit + return no_edit_link + + # Display a link to the admin page. + context = (capfirst(opts.verbose_name), admin_url, obj) + if account: + context += (_("services to delete:"),) + return format_html('{} {} {}', *context) + return format_html('{}: {}', *context) + else: + # Don't display link to edit, because it either has no + # admin or is edited inline. + return no_edit_link + + def format_nested(objs, result): + if isinstance(objs, list): + current = [] + for obj in objs: + format_nested(obj, current) + result.append(current) + else: + result.append(format(objs)) + + for nested in collector.nested(): + if isinstance(nested, list): + # Is lists of objects + current = [] + is_service = False + for service in nested: + if type(service) in registered_services: + if service == main_systemuser: + continue + current.append(format(service)) + to_delete.append(service) + is_service = True + elif is_service and isinstance(service, list): + nested = [] + format_nested(service, nested) + current.append(nested[0]) + is_service = False + else: + is_service = False + related_services.append(current) + elif isinstance(nested, modeladmin.model): + # Is account + # Prevent the deletion of the main system user, which will delete the account + main_systemuser = nested.main_systemuser + related_services.append(format(nested, account=True)) + + # The user has already confirmed the deletion. + # Do the deletion and return a None to display the change list view again. + if request.POST.get('post'): + accounts = len(queryset) + msg = _("Related services deleted and account disabled.") + for account in queryset: + account.is_active = False + account.save(update_fields=('is_active',)) + modeladmin.log_change(request, account, msg) + if accounts: + relateds = len(to_delete) + for obj in to_delete: + obj_display = force_str(obj) + modeladmin.log_deletion(request, obj, obj_display) + obj.delete() + context = { + 'accounts': accounts, + 'relateds': relateds, + } + msg = _("Successfully disabled %(accounts)d account and deleted %(relateds)d related services.") % context + modeladmin.message_user(request, msg, messages.SUCCESS) + # Return None to display the change list page again. + return None + + if len(queryset) == 1: + objects_name = force_str(opts.verbose_name) + else: + objects_name = force_str(opts.verbose_name_plural) + + model_count = {} + for model, objs in collector.model_objs.items(): + count = 0 + # discount main systemuser + if model is modeladmin.model.main_systemuser.field.related_model: + count = len(objs) - 1 + # Discount account + elif model is not modeladmin.model and model in registered_services: + count = len(objs) + if count: + model_count[model._meta.verbose_name_plural] = count + if not model_count: + modeladmin.message_user(request, _("Nothing to delete"), messages.WARNING) + return None + context = dict( + admin_site.each_context(request), + title=_("Are you sure?"), + objects_name=objects_name, + deletable_objects=[related_services], + model_count=dict(model_count).items(), + queryset=queryset, + opts=opts, + action_checkbox_name=helpers.ACTION_CHECKBOX_NAME, + ) + request.current_app = admin_site.name + # Display the confirmation page + template = 'admin/%s/%s/delete_related_services_confirmation.html' % (app_label, opts.model_name) + return TemplateResponse(request, template, context) +delete_related_services.short_description = _("Delete related services") + + +def disable_selected(modeladmin, request, queryset, disable=True): + opts = modeladmin.model._meta + app_label = opts.app_label + verbose_action_name = _("disabled") if disable else _("enabled") + # The user has already confirmed the deletion. + # Do the disable and return a None to display the change list view again. + if request.POST.get('post'): + n = 0 + for account in queryset: + account.disable() if disable else account.enable() + modeladmin.log_change(request, account, verbose_action_name.capitalize()) + n += 1 + modeladmin.message_user(request, ngettext( + _("One account has been successfully %s.") % verbose_action_name, + _("%i accounts have been successfully %s.") % (n, verbose_action_name), + n) + ) + return None + + user = request.user + admin_site = modeladmin.admin_site + + def format(obj): + has_admin = obj.__class__ in admin_site._registry + opts = obj._meta + no_edit_link = '%s: %s' % (capfirst(opts.verbose_name), force_str(obj)) + if has_admin: + try: + admin_url = reverse( + 'admin:%s_%s_change' % (opts.app_label, opts.model_name), + None, + (quote(obj._get_pk_val()),) + ) + except NoReverseMatch: + # Change url doesn't exist -- don't display link to edit + return no_edit_link + + p = '%s.%s' % (opts.app_label, get_permission_codename('delete', opts)) + if not user.has_perm(p): + perms_needed.add(opts.verbose_name) + # Display a link to the admin page. + context = (capfirst(opts.verbose_name), admin_url, obj) + return format_html('{}: {}', *context) + else: + # Don't display link to edit, because it either has no + # admin or is edited inline. + return no_edit_link + + display = [] + for account in queryset: + current = [] + for related in account.get_services_to_disable(): + current.append(format(related)) + display.append([format(account), current]) + + if len(queryset) == 1: + objects_name = force_str(opts.verbose_name) + else: + objects_name = force_str(opts.verbose_name_plural) + + context = dict( + admin_site.each_context(request), + action_name='disable_selected' if disable else 'enable_selected', + disable=disable, + title=_("Are you sure?"), + objects_name=objects_name, + deletable_objects=display, + queryset=queryset, + opts=opts, + action_checkbox_name=helpers.ACTION_CHECKBOX_NAME, + ) + request.current_app = admin_site.name + template = 'admin/%s/%s/disable_selected_confirmation.html' % (app_label, opts.model_name) + return TemplateResponse(request, template, context) +disable_selected.short_description = _("Disable selected accounts") +disable_selected.url_name = 'disable' +disable_selected.tool_description = _("Disable") + + +enable_selected = partial(disable_selected, disable=False) +enable_selected.__name__ = 'enable_selected' +enable_selected.url_name = 'enable' +enable_selected.tool_description = _("Enable") diff --git a/orchestra/contrib/accounts/admin.py b/orchestra/contrib/accounts/admin.py new file mode 100644 index 00000000..b2d6442a --- /dev/null +++ b/orchestra/contrib/accounts/admin.py @@ -0,0 +1,415 @@ +import copy +import re +from urllib.parse import parse_qsl + +from django import forms +from django.apps import apps +from django.urls import re_path as url +from django.contrib import admin, messages +from django.contrib.admin.utils import unquote +from django.contrib.auth import admin as auth +from django.urls import reverse +from django.http import HttpResponseRedirect +from django.templatetags.static import static +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin +from orchestra.admin.actions import SendEmail +from orchestra.admin.utils import wrap_admin_view, admin_link, set_url_query +from orchestra.contrib.services.settings import SERVICES_IGNORE_ACCOUNT_TYPE +from orchestra.core import services, accounts +from orchestra.forms import UserChangeForm +from orchestra.utils.apps import isinstalled + +from .actions import (list_contacts, service_report, delete_related_services, disable_selected, + enable_selected) +from .forms import AccountCreationForm +from .models import Account + + +class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin): + list_display = ('username', 'full_name', 'type', 'is_active') + list_filter = ( + 'type', 'is_active', + ) + add_fieldsets = ( + (_("User"), { + 'fields': ('username', 'password1', 'password2',), + }), + (_("Personal info"), { + 'fields': ('short_name', 'full_name', 'email', ('type', 'language'), 'comments'), + }), + (_("Permissions"), { + 'fields': ('is_superuser',) + }), + ) + fieldsets = ( + (_("User"), { + 'fields': ('username', 'password', 'main_systemuser_link') + }), + (_("Personal info"), { + 'fields': ('short_name', 'full_name', 'email', ('type', 'language'), 'comments'), + }), + (_("Permissions"), { + 'fields': ('is_superuser', 'is_active') + }), + (_("Important dates"), { + 'classes': ('collapse',), + 'fields': ('last_login', 'date_joined') + }), + ) + search_fields = ('username', 'short_name', 'full_name') + add_form = AccountCreationForm + form = UserChangeForm + filter_horizontal = () + change_readonly_fields = ('username', 'main_systemuser_link', 'is_active') + change_form_template = 'admin/accounts/account/change_form.html' + actions = ( + disable_selected, enable_selected, delete_related_services, list_contacts, service_report, + SendEmail() + ) + change_view_actions = (disable_selected, service_report, enable_selected) + ordering = () + + main_systemuser_link = admin_link('main_systemuser') + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'comments': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 4}) + return super(AccountAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): + if not add: + if request.method == 'GET' and not obj.is_active: + messages.warning(request, 'This account is disabled.') + context.update({ + 'services': sorted( + [model._meta for model in services.get() if model is not Account], + key=lambda i: i.verbose_name_plural.lower() + ), + 'accounts': sorted( + [model._meta for model in accounts.get() if model is not Account], + key=lambda i: i.verbose_name_plural.lower() + ) + }) + return super(AccountAdmin, self).render_change_form( + request, context, add, change, form_url, obj) + + def get_fieldsets(self, request, obj=None): + fieldsets = super(AccountAdmin, self).get_fieldsets(request, obj) + if not obj: + fields = AccountCreationForm.create_related_fields + if fields: + fieldsets = copy.deepcopy(fieldsets) + fieldsets = list(fieldsets) + fieldsets.insert(1, (_("Related services"), {'fields': fields})) + return fieldsets + + def save_model(self, request, obj, form, change): + if not change: + form.save_model(obj) + form.save_related(obj) + else: + if isinstalled('orchestra.contrib.orders') and isinstalled('orchestra.contrib.services'): + if 'type' in form.changed_data: + old_type = Account.objects.get(pk=obj.pk).type + new_type = form.cleaned_data['type'] + context = { + 'from': old_type.lower(), + 'to': new_type.lower(), + 'url': reverse('admin:orders_order_changelist'), + } + msg = '' + if old_type in SERVICES_IGNORE_ACCOUNT_TYPE and new_type not in SERVICES_IGNORE_ACCOUNT_TYPE: + context['url'] += '?account=%i&ignore=1' % obj.pk + msg = _("Account type has been changed from %(from)s to %(to)s. " + "You may want to mark existing ignored orders as not ignored.") + elif old_type not in SERVICES_IGNORE_ACCOUNT_TYPE and new_type in SERVICES_IGNORE_ACCOUNT_TYPE: + context['url'] += '?account=%i&ignore=0' % obj.pk + msg = _("Account type has been changed from %(from)s to %(to)s. " + "You may want to ignore existing not ignored orders.") + if msg: + messages.warning(request, mark_safe(msg % context)) + super(AccountAdmin, self).save_model(request, obj, form, change) + + def get_change_view_actions(self, obj=None): + views = super().get_change_view_actions(obj=obj) + if obj is not None: + if obj.is_active: + return [view for view in views if view.url_name != 'enable'] + return [view for view in views if view.url_name != 'disable'] + return views + + def get_actions(self, request): + actions = super().get_actions(request) + if 'delete_selected' in actions: + del actions['delete_selected'] + return actions + + +admin.site.register(Account, AccountAdmin) + + +class AccountListAdmin(AccountAdmin): + """ Account list to allow account selection when creating new services """ + list_display = ('select_account', 'username', 'type', 'username') + actions = None + change_list_template = 'admin/accounts/account/select_account_list.html' + + @mark_safe + def select_account(self, instance): + # TODO get query string from request.META['QUERY_STRING'] to preserve filters + context = { + 'url': '../?account=' + str(instance.pk), + 'name': instance.username, + 'plus': '+', + } + return _('%(plus)s Add to %(name)s') % context + select_account.short_description = _("account") + select_account.admin_order_field = 'username' + + def changelist_view(self, request, extra_context=None): + app_label = request.META['PATH_INFO'].split('/')[-5] + model = request.META['PATH_INFO'].split('/')[-4] + model = apps.get_model(app_label, model) + opts = model._meta + context = { + 'title': _("Select account for adding a new %s") % (opts.verbose_name), + 'original_opts': opts, + } + context.update(extra_context or {}) + response = super(AccountListAdmin, self).changelist_view(request, extra_context=context) + if hasattr(response, 'context_data'): + # user has submitted a change list change, we redirect directly to the add view + # if there is only one result + query = request.GET.get('q', '') + if query: + try: + account = Account.objects.get(username=query) + except Account.DoesNotExist: + pass + else: + return HttpResponseRedirect('../?account=%i' % account.pk) + queryset = response.context_data['cl'].queryset + if len(queryset) == 1: + return HttpResponseRedirect('../?account=%i' % queryset[0].pk) + return response + + +class AccountAdminMixin(object): + """ Provide basic account support to ModelAdmin and AdminInline classes """ + readonly_fields = ('account_link',) + filter_by_account_fields = [] + change_list_template = 'admin/accounts/account/change_list.html' + change_form_template = 'admin/accounts/account/change_form.html' + account = None + list_select_related = ('account',) + + @mark_safe + def display_active(self, instance): + if not instance.is_active: + return 'False' % static('admin/img/icon-no.svg') + elif not instance.account.is_active: + msg = _("Account disabled") + return 'False' % (static('admin/img/inline-delete.svg'), msg) + return 'False' % static('admin/img/icon-yes.svg') + display_active.short_description = _("active") + display_active.admin_order_field = 'is_active' + + def account_link(self, instance): + account = instance.account if instance.pk else self.account + return admin_link()(account) + account_link.short_description = _("account") + account_link.admin_order_field = 'account__username' + + def get_form(self, request, obj=None, **kwargs): + """ Warns user when object's account is disabled """ + form = super(AccountAdminMixin, self).get_form(request, obj, **kwargs) + try: + field = form.base_fields['is_active'] + except KeyError: + pass + else: + opts = self.model._meta + help_text = _( + "Designates whether this %(name)s should be treated as active. " + "Unselect this instead of deleting %(plural_name)s." + ) % { + 'name': opts.verbose_name, + 'plural_name': opts.verbose_name_plural, + } + if obj and not obj.account.is_active: + help_text += "
This user's account is dissabled" + field.help_text = _(help_text) + # Not available in POST + form.initial_account = self.get_changeform_initial_data(request).get('account') + return form + + def get_fields(self, request, obj=None): + """ remove account or account_link depending on the case """ + fields = super(AccountAdminMixin, self).get_fields(request, obj) + fields = list(fields) + if obj is not None or getattr(self, 'account_id', None): + try: + fields.remove('account') + except ValueError: + pass + else: + try: + fields.remove('account_link') + except ValueError: + pass + return fields + + def get_readonly_fields(self, request, obj=None): + """ provide account for filter_by_account_fields """ + if obj: + self.account = obj.account + return super(AccountAdminMixin, self).get_readonly_fields(request, obj) + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Filter by account """ + formfield = super(AccountAdminMixin, self).formfield_for_dbfield(db_field, **kwargs) + if db_field.name in self.filter_by_account_fields: + if self.account: + # Hack widget render in order to append ?account=id to the add url + old_render = formfield.widget.render + + def render(*args, **kwargs): + output = old_render(*args, **kwargs) + output = output.replace('/add/"', '/add/?account=%s"' % self.account.pk) + with_qargs = r'/add/?\1&account=%s"' % self.account.pk + output = re.sub(r'/add/\?([^".]*)"', with_qargs, output) + return mark_safe(output) + + formfield.widget.render = render + # Filter related object by account + formfield.queryset = formfield.queryset.filter(account=self.account) + # Apply heuristic order by + if not formfield.queryset.query.order_by: + related_fields = [f.name for f in db_field.related_model._meta.get_fields()] + if 'name' in related_fields: + formfield.queryset = formfield.queryset.order_by('name') + elif 'username' in related_fields: + formfield.queryset = formfield.queryset.order_by('username') + elif db_field.name == 'account': + if self.account: + formfield.initial = self.account.pk + elif Account.objects.count() == 1: + formfield.initial = 1 + formfield.queryset = formfield.queryset.order_by('username') + return formfield + + def get_formset(self, request, obj=None, **kwargs): + """ provides form.account for convinience """ + formset = super(AccountAdminMixin, self).get_formset(request, obj, **kwargs) + formset.form.account = self.account + formset.account = self.account + return formset + + def get_account_from_preserve_filters(self, request): + preserved_filters = self.get_preserved_filters(request) + preserved_filters = dict(parse_qsl(preserved_filters)) + cl_filters = preserved_filters.get('_changelist_filters') + if cl_filters: + return dict(parse_qsl(cl_filters)).get('account') + + def changeform_view(self, request, object_id=None, form_url='', extra_context=None): + account_id = self.get_account_from_preserve_filters(request) + if not object_id: + if account_id: + # Preselect account + set_url_query(request, 'account', account_id) + context = { + 'from_account': bool(account_id), + 'account': not account_id or Account.objects.get(pk=account_id), + 'account_opts': Account._meta, + } + context.update(extra_context or {}) + return super(AccountAdminMixin, self).changeform_view( + request, object_id, form_url=form_url, extra_context=context) + + def changelist_view(self, request, extra_context=None): + account_id = request.GET.get('account') + context = {} + if account_id: + opts = self.model._meta + account = Account.objects.get(pk=account_id) + context = { + 'account': not account_id or Account.objects.get(pk=account_id), + 'account_opts': Account._meta, + 'all_selected': True, + } + if not request.GET.get('all'): + context.update({ + 'all_selected': False, + 'title': _("Select %s to change for %s") % ( + opts.verbose_name, account.username), + }) + else: + request_copy = request.GET.copy() + request_copy.pop('account') + request.GET = request_copy + context.update(extra_context or {}) + return super(AccountAdminMixin, self).changelist_view(request, extra_context=context) + + +class SelectAccountAdminMixin(AccountAdminMixin): + """ Provides support for accounts on ModelAdmin """ + def get_inline_instances(self, request, obj=None): + inlines = super(AccountAdminMixin, self).get_inline_instances(request, obj) + if self.account: + account = self.account + else: + account = Account.objects.get(pk=request.GET['account']) + [setattr(inline, 'account', account) for inline in inlines] + return inlines + + def get_urls(self): + """ Hooks select account url """ + urls = super(AccountAdminMixin, self).get_urls() + admin_site = self.admin_site + opts = self.model._meta + info = opts.app_label, opts.model_name + account_list = AccountListAdmin(Account, admin_site).changelist_view + select_urls = [ + url("add/select-account/$", + wrap_admin_view(self, account_list), + name='%s_%s_select_account' % info), + ] + return select_urls + urls + + def add_view(self, request, form_url='', extra_context=None): + """ Redirects to select account view if required """ + if request.user.is_superuser: + from_account_id = self.get_account_from_preserve_filters(request) + if from_account_id: + set_url_query(request, 'account', from_account_id) + account_id = request.GET.get('account') + if account_id or Account.objects.count() == 1: + kwargs = {} + if account_id: + kwargs = dict(pk=account_id) + self.account = Account.objects.get(**kwargs) + opts = self.model._meta + context = { + 'title': _("Add %s for %s") % (opts.verbose_name, self.account.username), + 'from_account': bool(from_account_id), + 'from_select': True, + 'account': self.account, + 'account_opts': Account._meta, + } + context.update(extra_context or {}) + return super(AccountAdminMixin, self).add_view( + request, form_url=form_url, extra_context=context) + return HttpResponseRedirect('./select-account/?%s' % request.META['QUERY_STRING']) + + def save_model(self, request, obj, form, change): + """ + Given a model instance save it to the database. + """ + if not change: + obj.account_id = self.account.pk + obj.save() diff --git a/orchestra/contrib/accounts/api.py b/orchestra/contrib/accounts/api.py new file mode 100644 index 00000000..e602f908 --- /dev/null +++ b/orchestra/contrib/accounts/api.py @@ -0,0 +1,32 @@ +from django.utils.translation import gettext_lazy as _ +from rest_framework import viewsets, exceptions + +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin + +from .models import Account +from .serializers import AccountSerializer + + +class AccountApiMixin(object): + def get_queryset(self): + qs = super(AccountApiMixin, self).get_queryset() + return qs.filter(account=self.request.user.pk) + + +class AccountViewSet(LogApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): + queryset = Account.objects.all() + serializer_class = AccountSerializer + singleton_pk = lambda _,request: request.user.pk + + def get_queryset(self): + qs = super(AccountViewSet, self).get_queryset() + return qs.filter(id=self.request.user.pk) + + def destroy(self, request, pk=None): + # TODO reimplement in permissions + if not request.user.is_superuser: + raise exceptions.PermissionDenied(_("Accounts can not be deleted.")) + return super(AccountViewSet, self).destroy(request, pk=pk) + + +router.register(r'accounts', AccountViewSet) diff --git a/orchestra/contrib/accounts/apps.py b/orchestra/contrib/accounts/apps.py new file mode 100644 index 00000000..4501614c --- /dev/null +++ b/orchestra/contrib/accounts/apps.py @@ -0,0 +1,18 @@ +from django.apps import AppConfig +from django.db.models.signals import post_migrate +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import services, accounts + + +class AccountConfig(AppConfig): + name = 'orchestra.contrib.accounts' + verbose_name = _("Accounts") + + def ready(self): + from .management import create_initial_superuser + from .models import Account + services.register(Account, menu=False, dashboard=False) + accounts.register(Account, icon='Face-monkey.png') + post_migrate.connect(create_initial_superuser, + dispatch_uid="orchestra.contrib.accounts.management.createsuperuser") diff --git a/orchestra/contrib/accounts/filters.py b/orchestra/contrib/accounts/filters.py new file mode 100644 index 00000000..ff4be0ae --- /dev/null +++ b/orchestra/contrib/accounts/filters.py @@ -0,0 +1,27 @@ +from django.contrib.admin import SimpleListFilter +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + + +class IsActiveListFilter(SimpleListFilter): + title = _("is active") + parameter_name = 'active' + + def lookups(self, request, model_admin): + return ( + ('True', _("Yes")), + ('False', _("No")), + ('account', _("Account disabled")), + ('object', _("Object disabled")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(is_active=True, account__is_active=True) + elif self.value() == 'False': + return queryset.filter(Q(is_active=False) | Q(account__is_active=False)) + elif self.value() == 'account': + return queryset.filter(account__is_active=False) + elif self.value() == 'object': + return queryset.filter(is_active=False) + return queryset diff --git a/orchestra/contrib/accounts/forms.py b/orchestra/contrib/accounts/forms.py new file mode 100644 index 00000000..43689967 --- /dev/null +++ b/orchestra/contrib/accounts/forms.py @@ -0,0 +1,90 @@ +import logging +from collections import OrderedDict + +from django import forms +from django.core.exceptions import ValidationError +from django.apps import apps +from django.utils.translation import gettext_lazy as _ + +from orchestra.forms import UserCreationForm + +from . import settings +from .models import Account + + +logger = logging.getLogger(__name__) + + +def create_account_creation_form(): + fields = OrderedDict(**{ + 'enable_systemuser': forms.BooleanField(initial=True, required=False, + label=_("Enable systemuser"), + help_text=_("Designates whether to creates an enabled or disabled related system user. " + "Notice that a related system user will be always created.")) + }) + create_related = [] + for model, key, kwargs, help_text in settings.ACCOUNTS_CREATE_RELATED: + try: + model = apps.get_model(model) + except LookupError: + logger.error("%s not installed." % model) + else: + field_name = 'create_%s' % model._meta.model_name + label = _("Create %s") % model._meta.verbose_name + fields[field_name] = forms.BooleanField( + initial=True, required=False, label=label, help_text=help_text) + create_related.append((model, key, kwargs, help_text)) + + def clean(self, create_related=create_related): + """ unique usernames between accounts and system users """ + cleaned_data = UserCreationForm.clean(self) + try: + account = Account( + username=cleaned_data['username'], + password=cleaned_data['password1'] + ) + except KeyError: + # Previous validation error + return + errors = {} + systemuser_model = Account.main_systemuser.field.related_model + if systemuser_model.objects.filter(username=account.username).exists(): + errors['username'] = _("A system user with this name already exists.") + for model, key, related_kwargs, __ in create_related: + kwargs = { + key: eval(related_kwargs[key], {'account': account}) + } + if model.objects.filter(**kwargs).exists(): + verbose_name = model._meta.verbose_name + field_name = 'create_%s' % model._meta.model_name + errors[field_name] = ValidationError( + _("A %(type)s with this name already exists."), + params={'type': verbose_name}) + if errors: + raise ValidationError(errors) + + def save_model(self, account): + enable_systemuser=self.cleaned_data['enable_systemuser'] + account.save(active_systemuser=enable_systemuser) + + def save_related(self, account): + for model, key, related_kwargs, __ in settings.ACCOUNTS_CREATE_RELATED: + model = apps.get_model(model) + field_name = 'create_%s' % model._meta.model_name + if self.cleaned_data[field_name]: + kwargs = { + key: eval(value, {'account': account}) for key, value in related_kwargs.items() + } + model.objects.create(account=account, **kwargs) + + fields.update({ + 'create_related_fields': list(fields.keys()), + 'clean': clean, + 'save_model': save_model, + 'save_related': save_related, + }) + + return type('AccountCreationForm', (UserCreationForm,), fields) + + +AccountCreationForm = create_account_creation_form() diff --git a/orchestra/contrib/accounts/management/__init__.py b/orchestra/contrib/accounts/management/__init__.py new file mode 100644 index 00000000..c163afa3 --- /dev/null +++ b/orchestra/contrib/accounts/management/__init__.py @@ -0,0 +1,32 @@ +import sys +import textwrap + +from django.contrib.auth import get_user_model, base_user +from django.core.exceptions import FieldError +from django.core.management import execute_from_command_line +from django.db import models + + +def create_initial_superuser(**kwargs): + if '--noinput' not in sys.argv and '--fake' not in sys.argv and '--fake-initial' not in sys.argv: + model = get_user_model() + if not model.objects.filter(is_superuser=True).exists(): + sys.stdout.write(textwrap.dedent(""" + It appears that you just installed Accounts application. + You can now create a superuser: + + """) + ) + from ..models import Account + try: + Account.systemusers.field.model.objects.filter(account_id=1).exists() + except FieldError: + # avoid creating a systemuser when systemuser table is not ready + Account.save = models.Model.save + old_init = base_user.AbstractBaseUser.__init__ + def remove_is_staff(*args, **kwargs): + kwargs.pop('is_staff', None) + old_init(*args, **kwargs) + base_user.AbstractBaseUser.__init__ = remove_is_staff + manager = sys.argv[0] + execute_from_command_line(argv=[manager, 'createsuperuser']) diff --git a/orchestra/contrib/accounts/models.py b/orchestra/contrib/accounts/models.py new file mode 100644 index 00000000..b8c6a39a --- /dev/null +++ b/orchestra/contrib/accounts/models.py @@ -0,0 +1,207 @@ +from django.contrib.auth import models as auth +from django.conf import settings as djsettings +from django.core import validators +from django.db import models +from django.db.models import signals +from django.apps import apps +from django.utils import timezone, translation +from django.utils.translation import gettext_lazy as _ + +#from orchestra.contrib.orchestration.middlewares import OperationsMiddleware +#from orchestra.contrib.orchestration import Operation +from orchestra import core +from orchestra.models.utils import has_db_field +from orchestra.utils.mail import send_email_template + +from . import settings + + +class AccountManager(auth.UserManager): + def get_main(self): + return self.get(pk=settings.ACCOUNTS_MAIN_PK) + + +class Account(auth.AbstractBaseUser): + # Username max_length determined by LINUX system user/group lentgh: 32 + username = models.CharField(_("username"), max_length=32, unique=True, + help_text=_("Required. 32 characters or fewer. Letters, digits and ./-/_ only."), + validators=[ + validators.RegexValidator(r'^[\w.-]+$', _("Enter a valid username."), 'invalid') + ]) + main_systemuser = models.ForeignKey(settings.ACCOUNTS_SYSTEMUSER_MODEL, null=True, + related_name='accounts_main', editable=False, on_delete=models.SET_NULL) + short_name = models.CharField(_("short name"), max_length=64, blank=True) + full_name = models.CharField(_("full name"), max_length=256) + email = models.EmailField(_('email address'), help_text=_("Used for password recovery")) + type = models.CharField(_("type"), choices=settings.ACCOUNTS_TYPES, + max_length=32, default=settings.ACCOUNTS_DEFAULT_TYPE) + language = models.CharField(_("language"), max_length=2, + choices=settings.ACCOUNTS_LANGUAGES, + default=settings.ACCOUNTS_DEFAULT_LANGUAGE) + comments = models.TextField(_("comments"), max_length=256, blank=True) + is_superuser = models.BooleanField(_("superuser status"), default=False, + help_text=_("Designates that this user has all permissions without " + "explicitly assigning them.")) + is_active = models.BooleanField(_("active"), default=True, + help_text=_("Designates whether this account should be treated as active. " + "Unselect this instead of deleting accounts.")) + date_joined = models.DateTimeField(_("date joined"), default=timezone.now) + + objects = AccountManager() + + USERNAME_FIELD = 'username' + REQUIRED_FIELDS = ['email'] + + def __init__(self, *args, **kwargs): + # ignore `is_staff` kwarg because is handled with `is_superuser` + kwargs.pop('is_staff', None) + super().__init__(*args, **kwargs) + + def __str__(self): + return self.name + + @property + def name(self): + return self.username + + @property + def is_staff(self): + return self.is_superuser + + def save(self, active_systemuser=False, *args, **kwargs): + created = not self.pk + if not created: + was_active = Account.objects.filter(pk=self.pk).values_list('is_active', flat=True)[0] + super(Account, self).save(*args, **kwargs) + if created: + self.main_systemuser = self.systemusers.create( + account=self, username=self.username, password=self.password, + is_active=active_systemuser) + self.save(update_fields=('main_systemuser',)) + elif was_active != self.is_active: + self.notify_related() + + def clean(self): + self.short_name = self.short_name.strip() + self.full_name = self.full_name.strip() + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + self.notify_related() + + def enable(self): + self.is_active = True + self.save(update_fields=('is_active',)) + self.notify_related() + + def get_services_to_disable(self): + related_fields = [ + f for f in self._meta.get_fields() + if (f.one_to_many or f.one_to_one) + and f.auto_created and not f.concrete + ] + for rel in related_fields: + source = getattr(rel, 'related_model', rel.model) + if source in core.services and hasattr(source, 'active'): + for obj in getattr(self, rel.get_accessor_name()).all(): + yield obj + + def notify_related(self): + """ Trigger save() on related objects that depend on this account """ + for obj in self.get_services_to_disable(): + signals.pre_save.send(sender=type(obj), instance=obj) + signals.post_save.send(sender=type(obj), instance=obj) +# OperationsMiddleware.collect(Operation.SAVE, instance=obj, update_fields=()) + + def get_contacts_emails(self, usages=None): + contacts = self.contacts.all() + if usages is not None: + contactes = contacts.filter(email_usages=usages) + return contacts.values_list('email', flat=True) + + def send_email(self, template, context, email_from=None, usages=None, attachments=[], html=None): + contacts = self.contacts.filter(email_usages=usages) + email_to = self.get_contacts_emails(usages) + extra_context = { + 'account': self, + 'email_from': email_from or djsettings.SERVER_EMAIL, + } + extra_context.update(context) + with translation.override(self.language): + send_email_template(template, extra_context, email_to, email_from=email_from, + html=html, attachments=attachments) + + def get_full_name(self): + return self.full_name or self.short_name or self.username + + def get_short_name(self): + """ Returns the short name for the user """ + return self.short_name or self.username or self.full_name + + def has_perm(self, perm, obj=None): + """ + Returns True if the user has the specified permission. This method + queries all available auth backends, but returns immediately if any + backend returns True. Thus, a user who has permission from a single + auth backend is assumed to have permission in general. If an object is + provided, permissions for this specific object are checked. + applabel.action_modelname + """ + if not self.is_active: + return False + # Active superusers have all permissions. + if self.is_superuser: + return True + app, action_model = perm.split('.') + action, model = action_model.split('_', 1) + service_apps = set(model._meta.app_label for model in core.services.get().keys()) + accounting_apps = set(model._meta.app_label for model in core.accounts.get().keys()) + import inspect + if ((app in service_apps or (action == 'view' and app in accounting_apps))): + # class-level permissions + if inspect.isclass(obj): + return True + elif obj and getattr(obj, 'account', None) == self: + return True + + def has_perms(self, perm_list, obj=None): + """ + Returns True if the user has each of the specified permissions. If + object is passed, it checks if the user has all required perms for this + object. + """ + for perm in perm_list: + if not self.has_perm(perm, obj): + return False + return True + + def has_module_perms(self, app_label): + """ + Returns True if the user has any permissions in the given app label. + Uses pretty much the same logic as has_perm, above. + """ + # Active superusers have all permissions. + if self.is_active and self.is_superuser: + return True + + def get_related_passwords(self, db_field=False): + related = [ + self.main_systemuser, + ] + for model, key, related_kwargs, __ in settings.ACCOUNTS_CREATE_RELATED: + if 'password' not in related_kwargs: + continue + model = apps.get_model(model) + kwargs = { + key: eval(related_kwargs[key], {'account': self}) + } + try: + rel = model.objects.get(account=self, **kwargs) + except model.DoesNotExist: + continue + if db_field: + if not has_db_field(rel, 'password'): + continue + related.append(rel) + return related diff --git a/orchestra/contrib/accounts/serializers.py b/orchestra/contrib/accounts/serializers.py new file mode 100644 index 00000000..9f05b167 --- /dev/null +++ b/orchestra/contrib/accounts/serializers.py @@ -0,0 +1,27 @@ +from rest_framework import serializers + +from .models import Account + + +class AccountSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Account + fields = ( + 'url', 'id', 'username', 'type', 'language', 'short_name', 'full_name', 'date_joined', 'last_login', + 'is_active' + ) + + +class AccountSerializerMixin(object): + def __init__(self, *args, **kwargs): + super(AccountSerializerMixin, self).__init__(*args, **kwargs) + self.account = self.get_account() + + def get_account(self): + request = self.context.get('request') + if request: + return request.user + + def create(self, validated_data): + validated_data['account'] = self.get_account() + return super(AccountSerializerMixin, self).create(validated_data) diff --git a/orchestra/contrib/accounts/settings.py b/orchestra/contrib/accounts/settings.py new file mode 100644 index 00000000..b8a25944 --- /dev/null +++ b/orchestra/contrib/accounts/settings.py @@ -0,0 +1,74 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN + + +ACCOUNTS_TYPES = Setting('ACCOUNTS_TYPES', + ( + ('INDIVIDUAL', _("Individual")), + ('ASSOCIATION', _("Association")), + ('CUSTOMER', _("Customer")), + ('COMPANY', _("Company")), + ('PUBLICBODY', _("Public body")), + ('STAFF', _("Staff")), + ('FRIEND', _("Friend")), + ), + validators=[Setting.validate_choices] +) + + +ACCOUNTS_DEFAULT_TYPE = Setting('ACCOUNTS_DEFAULT_TYPE', + 'INDIVIDUAL', choices=ACCOUNTS_TYPES) + + +ACCOUNTS_LANGUAGES = Setting('ACCOUNTS_LANGUAGES', + ( + ('EN', _('English')), + ), + validators=[Setting.validate_choices] +) + + +ACCOUNTS_DEFAULT_LANGUAGE = Setting('ACCOUNTS_DEFAULT_LANGUAGE', + 'EN', + choices=ACCOUNTS_LANGUAGES +) + + +ACCOUNTS_SYSTEMUSER_MODEL = Setting('ACCOUNTS_SYSTEMUSER_MODEL', + 'systemusers.SystemUser', + validators=[Setting.validate_model_label], +) + + +ACCOUNTS_MAIN_PK = Setting('ACCOUNTS_MAIN_PK', + 1 +) + + +ACCOUNTS_CREATE_RELATED = Setting('ACCOUNTS_CREATE_RELATED', + ( + # , , , + ('mailboxes.Mailbox', + 'name', + { + 'name': 'account.username', + 'password': 'account.password', + }, + _("Designates whether to creates a related mailbox with the same name and password or not."), + ), + ('domains.Domain', + 'name', + { + 'name': '"%s.{}" % account.username.replace("_", "-")'.format(ORCHESTRA_BASE_DOMAIN), + }, + _("Designates whether to creates a related subdomain <username>.{} or not.".format(ORCHESTRA_BASE_DOMAIN)), + ), + ), +) + + +ACCOUNTS_SERVICE_REPORT_TEMPLATE = Setting('ACCOUNTS_SERVICE_REPORT_TEMPLATE', + 'admin/accounts/account/service_report.html' +) diff --git a/orchestra/contrib/accounts/templates/admin/accounts/account/change_form.html b/orchestra/contrib/accounts/templates/admin/accounts/account/change_form.html new file mode 100644 index 00000000..b1a41386 --- /dev/null +++ b/orchestra/contrib/accounts/templates/admin/accounts/account/change_form.html @@ -0,0 +1,42 @@ +{% extends "orchestra/admin/change_form.html" %} +{% load i18n admin_urls static admin_modify %} + + +{% block breadcrumbs %} +
+{% trans 'Home' %} +{% if from_account %} +› {{ account_opts.app_config.verbose_name }} +› {{ account_opts.verbose_name_plural|capfirst }} +› {{ account|truncatewords:"18" }} +{% else %} +› {{ opts.app_config.verbose_name }} +› {% if has_change_permission %}{{ opts.verbose_name_plural|capfirst }}{% else %}{{ opts.verbose_name_plural|capfirst }}{% endif %} +{% endif %} +{% if from_select %} +› {% blocktrans with name=original_opts.verbose_name %}Select {{ name }} account{% endblocktrans %} +{% endif %} +› {% if add %}{% trans 'Add' %} {{ opts.verbose_name }}{% else %}{{ original|truncatewords:"18" }}{% endif %} +
+{% endblock %} + + +{% block object-tools-items %} +{% if services %} +
    • +{% endif %} +{% if accounts %} +
    • +{% endif %} +{{ block.super }} +{% endblock %} diff --git a/orchestra/contrib/accounts/templates/admin/accounts/account/change_list.html b/orchestra/contrib/accounts/templates/admin/accounts/account/change_list.html new file mode 100644 index 00000000..08a7b563 --- /dev/null +++ b/orchestra/contrib/accounts/templates/admin/accounts/account/change_list.html @@ -0,0 +1,49 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls admin_list %} + + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +{% if account %} +› {{ account_opts.app_config.verbose_name }} +› {{ account_opts.verbose_name_plural|capfirst }} +› {{ account|truncatewords:"18" }} +{% else %} +› {{ cl.opts.app_config.verbose_name }} +{% endif %} +› {{ cl.opts.verbose_name_plural|capfirst }} +
    +{% endblock %} + + +{% block object-tools-items %} +
  • + {% url cl.opts|admin_urlname:'add' as add_url %} + + {% if all_selected %} + {% blocktrans with cl.opts.verbose_name as name %}Add {{ name }}{% endblocktrans %} + {% else %} + {% blocktrans with cl.opts.verbose_name as name and account|truncatewords:"18" as account %}Add {{ account }} {{ name }}{% endblocktrans %} + {% endif %} + +
    • +{% endblock %} + + +{% block filters %} + {% if cl.has_filters %} +
    +

    {% trans 'Filter' %}

    + {% if account %} +

    {% trans 'By account' %}

    + + {% endif %} + {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %} +
    + {% endif %} +{% endblock %} + diff --git a/orchestra/contrib/accounts/templates/admin/accounts/account/delete_related_services_confirmation.html b/orchestra/contrib/accounts/templates/admin/accounts/account/delete_related_services_confirmation.html new file mode 100644 index 00000000..b25aee0b --- /dev/null +++ b/orchestra/contrib/accounts/templates/admin/accounts/account/delete_related_services_confirmation.html @@ -0,0 +1,39 @@ +{% extends "admin/delete_selected_confirmation.html" %} +{% load i18n l10n admin_urls %} + +{% block content %} +{% if perms_lacking %} +

    {% blocktrans %}Deleting the selected {{ objects_name }} would result in deleting related objects, but your account doesn't have permission to delete the following types of objects:{% endblocktrans %}

    +
      + {% for obj in perms_lacking %} +
    • {{ obj }}
      • + {% endfor %} +
    +{% elif protected %} +

    {% blocktrans %}Deleting the selected {{ objects_name }} would require deleting the following protected related objects:{% endblocktrans %}

    +
      + {% for obj in protected %} +
    • {{ obj }}
      • + {% endfor %} +
    +{% else %} +

    {% blocktrans %}Are you sure you want to delete the selected {{ objects_name }}? All of the following objects and their related items will be deleted:{% endblocktrans %}

    + {% include "admin/includes/object_delete_summary.html" %} +

    {% trans "Objects" %}

    + {% for deletable_object in deletable_objects %} +
      {{ deletable_object|unordered_list }}
    + {% endfor %} +
    {% csrf_token %} +
    + {% for obj in queryset %} + + {% endfor %} + + + + {% trans "No, take me back" %} +
    +
    +{% endif %} +{% endblock %} + diff --git a/orchestra/contrib/accounts/templates/admin/accounts/account/disable_selected_confirmation.html b/orchestra/contrib/accounts/templates/admin/accounts/account/disable_selected_confirmation.html new file mode 100644 index 00000000..7ada7249 --- /dev/null +++ b/orchestra/contrib/accounts/templates/admin/accounts/account/disable_selected_confirmation.html @@ -0,0 +1,35 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% if disable%}{% blocktrans %}Disable {{ objects_name }}{% endblocktrans %}{% else %}{% blocktrans %}Enable {{ objects_name }}{% endblocktrans %}{% endif %} +
    +{% endblock %} + +{% block content %} +{% if disable%}

    {% blocktrans %}Are you sure you want to disable selected {{ objects_name }}?{% endblocktrans %}

    +{% else %}

    {% blocktrans %}Are you sure you want to enable selected {{ objects_name }}?{% endblocktrans %}

    +{% endif %} +

    {% trans "Objects" %}

    +{% for deletable_object in deletable_objects %} +
      {{ deletable_object|unordered_list }}
    +{% endfor %} +
    {% csrf_token %} +
    +{% for obj in queryset %} + +{% endfor %} + + + +{% trans "No, take me back" %} +
    +
    +{% endblock %} + diff --git a/orchestra/contrib/accounts/templates/admin/accounts/account/select_account_list.html b/orchestra/contrib/accounts/templates/admin/accounts/account/select_account_list.html new file mode 100644 index 00000000..bb60f3ad --- /dev/null +++ b/orchestra/contrib/accounts/templates/admin/accounts/account/select_account_list.html @@ -0,0 +1,13 @@ +{% extends 'admin/change_list.html' %} +{% load i18n admin_urls %} + + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› {{ original_opts.app_config.verbose_name }} +› {{ original_opts.verbose_name_plural|capfirst }} +› {% blocktrans with name=original_opts.verbose_name %}Select {{ name }} account{% endblocktrans %} +
    +{% endblock %} + diff --git a/orchestra/contrib/accounts/templates/admin/accounts/account/service_report.html b/orchestra/contrib/accounts/templates/admin/accounts/account/service_report.html new file mode 100644 index 00000000..f10d116f --- /dev/null +++ b/orchestra/contrib/accounts/templates/admin/accounts/account/service_report.html @@ -0,0 +1,84 @@ +{% load i18n admin_urls utils %} + + + {% block title %}Account service report{% endblock %} + + {% block head %}{% endblock %} + + + + +
    {% trans "Service report generated on" %} {{ date | date }}
    +{% for account, items in accounts %} +

    {{ account.get_full_name }} - {{ account.username }}

    +
    + +{% endfor %} + + diff --git a/orchestra/contrib/bills/__init__.py b/orchestra/contrib/bills/__init__.py new file mode 100644 index 00000000..c568ce60 --- /dev/null +++ b/orchestra/contrib/bills/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.bills.apps.BillsConfig' diff --git a/orchestra/contrib/bills/actions.py b/orchestra/contrib/bills/actions.py new file mode 100644 index 00000000..64c9ed57 --- /dev/null +++ b/orchestra/contrib/bills/actions.py @@ -0,0 +1,377 @@ +import io +import zipfile +from datetime import date + +from django.contrib import messages +from django.contrib.admin import helpers +from django.core.exceptions import ValidationError +from django.urls import reverse +from django.db import transaction +from django.forms.models import modelformset_factory +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render, redirect +from django.utils import translation, timezone +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.admin.decorators import action_with_confirmation +from orchestra.admin.forms import AdminFormSet +from orchestra.admin.utils import get_object_from_url, change_url + +from . import settings +from .forms import SelectSourceForm +from .helpers import validate_contact, set_context_emails +from .models import Bill, BillLine + + +def view_bill(modeladmin, request, queryset): + bill = queryset.get() + if not validate_contact(request, bill): + return + html = bill.html or bill.render() + return HttpResponse(html) +view_bill.tool_description = _("View") +view_bill.url_name = 'view' +view_bill.hidden = True + + +@transaction.atomic +def close_bills(modeladmin, request, queryset, action='close_bills'): + # Validate bills + for bill in queryset: + if not validate_contact(request, bill): + return False + if not bill.is_open: + messages.warning(request, _("Selected bills should be in open state")) + return False + SelectSourceFormSet = modelformset_factory(modeladmin.model, form=SelectSourceForm, formset=AdminFormSet, extra=0) + formset = SelectSourceFormSet(queryset=queryset) + if request.POST.get('post') == 'generic_confirmation': + formset = SelectSourceFormSet(request.POST, request.FILES, queryset=queryset) + if formset.is_valid(): + transactions = [] + for form in formset.forms: + source = form.cleaned_data['source'] + transaction = form.instance.close(payment=source) + if transaction: + transactions.append(transaction) + for bill in queryset: + modeladmin.log_change(request, bill, 'Closed') + messages.success(request, _("Selected bills have been closed")) + if transactions: + num = len(transactions) + if num == 1: + url = change_url(transactions[0]) + else: + url = reverse('admin:payments_transaction_changelist') + url += 'id__in=%s' % ','.join([str(t.id) for t in transactions]) + context = { + 'url': url, + 'num': num, + } + message = ngettext( + _('One related transaction has been created') % context, + _('%(num)i related transactions have been created') % context, + num) + messages.success(request, mark_safe(message)) + return + opts = modeladmin.model._meta + context = { + 'title': _("Are you sure about closing the following bills?"), + 'content_message': _("Once a bill is closed it can not be further modified.

    " + "

    Please select a payment source for the selected bills"), + 'action_name': 'Close bills', + 'action_value': action, + 'display_objects': [], + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'formset': formset, + 'obj': get_object_from_url(modeladmin, request), + } + template = 'admin/orchestra/generic_confirmation.html' + if action == 'close_send_download_bills': + template = 'admin/bills/bill/close_send_download_bills.html' + return render(request, template, context) +close_bills.tool_description = _("Close") +close_bills.url_name = 'close' + + +def send_bills_action(modeladmin, request, queryset): + """ + raw function without confirmation + enables reuse on close_send_download_bills because of generic_confirmation.action_view + """ + for bill in queryset: + if not validate_contact(request, bill): + return False + num = 0 + for bill in queryset: + bill.send() + modeladmin.log_change(request, bill, 'Sent') + num += 1 + messages.success(request, ngettext( + _("One bill has been sent."), + _("%i bills have been sent.") % num, + num)) + + +@action_with_confirmation(extra_context=set_context_emails) +def send_bills(modeladmin, request, queryset): + return send_bills_action(modeladmin, request, queryset) +send_bills.verbose_name = lambda bill: _("Resend" if getattr(bill, 'is_sent', False) else "Send") +send_bills.url_name = 'send' + + +def download_bills(modeladmin, request, queryset): + for bill in queryset: + if not validate_contact(request, bill): + return False + if len(queryset) > 1: + bytesio = io.BytesIO() + archive = zipfile.ZipFile(bytesio, 'w') + for bill in queryset: + pdf = bill.as_pdf() + archive.writestr('%s.pdf' % bill.number, pdf) + archive.close() + response = HttpResponse(bytesio.getvalue(), content_type='application/zip') + response['Content-Disposition'] = 'attachment; filename="orchestra-bills.zip"' + return response + bill = queryset[0] + pdf = bill.as_pdf() + response = HttpResponse(pdf, content_type='application/pdf') + response['Content-Disposition'] = 'attachment; filename="%s.pdf"' % bill.number + return response +download_bills.tool_description = _("Download") +download_bills.url_name = 'download' + + +def close_send_download_bills(modeladmin, request, queryset): + response = close_bills(modeladmin, request, queryset, action='close_send_download_bills') + if response is False: + # Not a valid contact or closed bill + return + if request.POST.get('post') == 'generic_confirmation': + response = send_bills_action(modeladmin, request, queryset) + if response is False: + # Not a valid contact + return + return download_bills(modeladmin, request, queryset) + return response +close_send_download_bills.tool_description = _("C.S.D.") +close_send_download_bills.url_name = 'close-send-download' +close_send_download_bills.help_text = _("Close, send and download bills in one shot.") + + +def manage_lines(modeladmin, request, queryset): + url = reverse('admin:bills_bill_manage_lines') + url += '?ids=%s' % ','.join(map(str, queryset.values_list('id', flat=True))) + return redirect(url) + + +@action_with_confirmation() +def undo_billing(modeladmin, request, queryset): + group = {} + for line in queryset.select_related('order'): + if line.order_id: + try: + group[line.order].append(line) + except KeyError: + group[line.order] = [line] + + # Validate + for order, lines in group.items(): + prev = None + billed_on = date.max + billed_until = date.max + for line in sorted(lines, key=lambda l: l.start_on): + if billed_on is not None: + if line.order_billed_on is None: + billed_on = line.order_billed_on + else: + billed_on = min(billed_on, line.order_billed_on) + if billed_until is not None: + if line.order_billed_until is None: + billed_until = line.order_billed_until + else: + billed_until = min(billed_until, line.order_billed_until) + if prev: + if line.start_on != prev: + messages.error(request, "Line dates doesn't match.") + return + else: + # First iteration + if order.billed_on < line.start_on: + messages.error(request, "Billed on is smaller than first line start_on.") + return + prev = line.end_on + nlines += 1 + if not prev: + messages.error(request, "Order does not have lines!.") + order.billed_until = billed_until + order.billed_on = billed_on + + # Commit changes + norders, nlines = 0, 0 + for order, lines in group.items(): + for line in lines: + nlines += 1 + line.delete() + # TODO update order history undo billing + order.save(update_fields=('billed_until', 'billed_on')) + norders += 1 + + messages.success(request, _("%(norders)s orders and %(nlines)s lines undoed.") % { + 'nlines': nlines, + 'norders': norders + }) + + +def move_lines(modeladmin, request, queryset, action=None): + # Validate + target = request.GET.get('target') + if not target: + # select target + context = {} + return render(request, 'admin/orchestra/generic_confirmation.html', context) + target = Bill.objects.get(pk=int(pk)) + if request.POST.get('post') == 'generic_confirmation': + for line in queryset: + line.bill = target + line.save(update_fields=['bill']) + # TODO bill history update + messages.success(request, _("Lines moved")) + # Final confirmation + return render(request, 'admin/orchestra/generic_confirmation.html', context) + + +def copy_lines(modeladmin, request, queryset): + # same as move, but changing action behaviour + return move_lines(modeladmin, request, queryset) + + +def validate_amend_bills(bills): + for bill in bills: + if bill.is_open: + raise ValidationError(_("Selected bills should be in closed state")) + if bill.type not in bill.AMEND_MAP: + raise ValidationError(_("%s can not be amended.") % bill.get_type_display()) + + +@action_with_confirmation(validator=validate_amend_bills) +def amend_bills(modeladmin, request, queryset): + amend_ids = [] + for bill in queryset: + with translation.override(bill.account.language): + amend_type = bill.get_amend_type() + context = { + 'related_type': _(bill.get_type_display()), + 'number': bill.number, + 'date': bill.created_on, + } + amend = Bill.objects.create( + account=bill.account, + type=amend_type, + amend_of=bill, + ) + context['type'] = _(amend.get_type_display()) + amend.comments = _("%(type)s of %(related_type)s %(number)s and creation date %(date)s") % context + amend.save(update_fields=('comments',)) + for tax, subtotals in bill.compute_subtotals().items(): + context['tax'] = tax + line = BillLine.objects.create( + bill=amend, + start_on=bill.created_on, + description=_("%(related_type)s %(number)s subtotal for tax %(tax)s%%") % context, + subtotal=subtotals[0], + tax=tax + ) + amend_ids.append(amend.pk) + modeladmin.log_change(request, bill, 'Amended, amend id is %i' % amend.id) + num = len(amend_ids) + if num == 1: + amend_url = reverse('admin:bills_bill_change', args=amend_ids) + else: + amend_url = reverse('admin:bills_bill_changelist') + amend_url += '?id=%s' % ','.join(map(str, amend_ids)) + context = { + 'url': amend_url, + 'num': num, + } + messages.success(request, mark_safe(ngettext( + _('One amendment bill have been generated.') % context, + _('%(num)i amendment bills have been generated.') % context, + num + ))) +amend_bills.tool_description = _("Amend") +amend_bills.url_name = 'amend' + + +def bill_report(modeladmin, request, queryset): + subtotals = {} + total = 0 + for bill in queryset: + for tax, subtotal in bill.compute_subtotals().items(): + try: + subtotals[tax][0] += subtotal[0] + except KeyError: + subtotals[tax] = subtotal + else: + subtotals[tax][1] += subtotal[1] + total += bill.compute_total() + context = { + 'subtotals': subtotals, + 'total': total, + 'bills': queryset, + 'currency': settings.BILLS_CURRENCY, + } + return render(request, 'admin/bills/bill/report.html', context) + + +def service_report(modeladmin, request, queryset): + services = {} + totals = [0, 0, 0, 0, 0] + now = timezone.now().date() + if queryset.model == Bill: + queryset = BillLine.objects.filter(bill_id__in=queryset.values_list('id', flat=True)) + # Filter amends + queryset = queryset.filter(bill__amend_of__isnull=True) + for line in queryset.select_related('order__service').prefetch_related('sublines'): + order, service = None, None + if line.order_id: + order = line.order + service = order.service + name = service.description + active, cancelled = (1, 0) if not order.cancelled_on or order.cancelled_on > now else (0, 1) + nominal_price = order.service.nominal_price + else: + name = '*%s' % line.description + active = 1 + cancelled = 0 + nominal_price = 0 + try: + info = services[name] + except KeyError: + info = [active, cancelled, nominal_price, line.quantity or 1, line.compute_total()] + services[name] = info + else: + info[0] += active + info[1] += cancelled + info[3] += line.quantity or 1 + info[4] += line.compute_total() + totals[0] += active + totals[1] += cancelled + totals[2] += nominal_price + totals[3] += line.quantity or 1 + totals[4] += line.compute_total() + context = { + 'services': sorted(services.items(), key=lambda n: -n[1][4]), + 'totals': totals, + } + return render(request, 'admin/bills/billline/report.html', context) + + +def list_bills(modeladmin, request, queryset): + ids = ','.join(map(str, queryset.values_list('bill_id', flat=True).distinct())) + return HttpResponseRedirect('../bill/?id__in=%s' % ids) diff --git a/orchestra/contrib/bills/admin.py b/orchestra/contrib/bills/admin.py new file mode 100644 index 00000000..ac453dbe --- /dev/null +++ b/orchestra/contrib/bills/admin.py @@ -0,0 +1,493 @@ +from django import forms +from django.urls import re_path as url +from django.contrib import admin, messages +from django.contrib.admin.utils import unquote +from django.urls import reverse +from django.db import models +from django.db.models import F, Sum, Prefetch +from django.db.models.functions import Coalesce +from django.templatetags.static import static +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ +from django.shortcuts import redirect + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import admin_date, insertattr, admin_link, change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin, AccountAdmin +from orchestra.forms.widgets import PaddingCheckboxSelectMultiple + +from . import settings, actions +from .filters import (BillTypeListFilter, HasBillContactListFilter, TotalListFilter, + PaymentStateListFilter, AmendedListFilter) +from .models import (Bill, Invoice, AmendmentInvoice, AbonoInvoice, Fee, AmendmentFee, ProForma, BillLine, + BillSubline, BillContact) + + +PAYMENT_STATE_COLORS = { + Bill.OPEN: 'grey', + Bill.CREATED: 'magenta', + Bill.PROCESSED: 'darkorange', + Bill.AMENDED: 'blue', + Bill.PAID: 'green', + Bill.EXECUTED: 'olive', + Bill.BAD_DEBT: 'red', + Bill.INCOMPLETE: 'red', +} + + +class BillSublineInline(admin.TabularInline): + model = BillSubline + fields = ('description', 'total', 'type') + + def get_readonly_fields(self, request, obj=None): + fields = super().get_readonly_fields(request, obj) + if obj and not obj.bill.is_open: + return self.get_fields(request) + return fields + + def get_max_num(self, request, obj=None): + if obj and not obj.bill.is_open: + return 0 + return super().get_max_num(request, obj) + + def has_delete_permission(self, request, obj=None): + if obj and not obj.bill.is_open: + return False + return super().has_delete_permission(request, obj) + + +class BillLineInline(admin.TabularInline): + model = BillLine + fields = ( + 'description', 'order_link', 'start_on', 'end_on', 'rate', 'quantity', 'tax', + 'subtotal', 'display_total', + ) + readonly_fields = ('display_total', 'order_link') + + order_link = admin_link('order', display='pk') + + @mark_safe + def display_total(self, line): + if line.pk: + total = line.compute_total() + sublines = line.sublines.all() + url = change_url(line) + if sublines: + content = '\n'.join(['%s: %s' % (sub.description, sub.total) for sub in sublines]) + img = static('admin/img/icon-alert.svg') + return '%s ' % (url, content, total, img) + return '%s' % (url, total) + display_total.short_description = _("Total") + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'description': + kwargs['widget'] = forms.TextInput(attrs={'size':'50'}) + elif db_field.name not in ('start_on', 'end_on'): + kwargs['widget'] = forms.TextInput(attrs={'size':'6'}) + return super().formfield_for_dbfield(db_field, **kwargs) + + def get_queryset(self, request): + qs = super().get_queryset(request) + return qs.prefetch_related('sublines').select_related('order') + + +class ClosedBillLineInline(BillLineInline): + # TODO reimplement as nested inlines when upstream + # https://code.djangoproject.com/ticket/9025 + + fields = ( + 'display_description', 'order_link', 'start_on', 'end_on', 'rate', 'quantity', 'tax', + 'display_subtotal', 'display_total' + ) + readonly_fields = fields + can_delete = False + + @mark_safe + def display_description(self, line): + descriptions = [line.description] + for subline in line.sublines.all(): + descriptions.append(' ' * 4 + subline.description) + return '
    '.join(descriptions) + display_description.short_description = _("Description") + + @mark_safe + def display_subtotal(self, line): + subtotals = [' ' + str(line.subtotal)] + for subline in line.sublines.all(): + subtotals.append(str(subline.total)) + return '
    '.join(subtotals) + display_subtotal.short_description = _("Subtotal") + + def display_total(self, line): + if line.pk: + return line.compute_total() + display_total.short_description = _("Total") + + def has_add_permission(self, request, obj): + return False + + +class BillLineAdmin(admin.ModelAdmin): + list_display = ( + 'description', 'bill_link', 'display_is_open', 'account_link', 'rate', 'quantity', + 'tax', 'subtotal', 'display_sublinetotal', 'display_total' + ) + actions = ( + actions.undo_billing, actions.move_lines, actions.copy_lines, actions.service_report, + actions.list_bills, + ) + fieldsets = ( + (None, { + 'fields': ('bill_link', 'description', 'tax', 'start_on', 'end_on', 'amended_line_link') + }), + (_("Totals"), { + 'fields': ('rate', ('quantity', 'verbose_quantity'), 'subtotal', 'display_sublinetotal', + 'display_total'), + }), + (_("Order"), { + 'fields': ('order_link', 'order_billed_on', 'order_billed_until',) + }), + ) + readonly_fields = ( + 'bill_link', 'order_link', 'amended_line_link', 'display_sublinetotal', 'display_total' + ) + list_filter = ('tax', 'bill__is_open', 'order__service') + list_select_related = ('bill', 'bill__account') + search_fields = ('description', 'bill__number') + inlines = (BillSublineInline,) + + account_link = admin_link('bill__account') + bill_link = admin_link('bill') + order_link = admin_link('order') + amended_line_link = admin_link('amended_line') + + def display_is_open(self, instance): + return instance.bill.is_open + display_is_open.short_description = _("Is open") + display_is_open.boolean = True + + def display_sublinetotal(self, instance): + total = instance.subline_total + return total if total is not None else '---' + display_sublinetotal.short_description = _("Sublines") + display_sublinetotal.admin_order_field = 'subline_total' + + def display_total(self, instance): + return round(instance.computed_total or 0, 2) + display_total.short_description = _("Total") + display_total.admin_order_field = 'computed_total' + + def get_readonly_fields(self, request, obj=None): + fields = super().get_readonly_fields(request, obj) + if obj and not obj.bill.is_open: + return list(fields) + [ + 'description', 'tax', 'start_on', 'end_on', 'rate', 'quantity', 'verbose_quantity', + 'subtotal', 'order_billed_on', 'order_billed_until' + ] + return fields + + def get_queryset(self, request): + qs = super().get_queryset(request) + qs = qs.annotate( + subline_total=Sum('sublines__total'), + computed_total=(F('subtotal') + Sum(Coalesce('sublines__total', 0))) * (1+F('tax')/100), + ) + return qs + + def has_delete_permission(self, request, obj=None): + if obj and not obj.bill.is_open: + return False + return super().has_delete_permission(request, obj) + + +class BillLineManagerAdmin(BillLineAdmin): + def get_queryset(self, request): + qset = super().get_queryset(request) + if self.bill_ids: + return qset.filter(bill_id__in=self.bill_ids) + return qset + + def changelist_view(self, request, extra_context=None): + GET_copy = request.GET.copy() + bill_ids = GET_copy.pop('ids', None) + if bill_ids: + bill_ids = bill_ids[0] + request.GET = GET_copy + bill_ids = list(map(int, bill_ids.split(','))) + else: + messages.error(request, _("No bills selected.")) + return redirect('..') + self.bill_ids = bill_ids + bill = None + if len(bill_ids) == 1: + bill_url = reverse('admin:bills_bill_change', args=(bill_ids[0],)) + bill = Bill.objects.get(pk=bill_ids[0]) + bill_link = '%s' % (bill_url, bill.number) + title = mark_safe(_("Manage %s bill lines") % bill_link) + if not bill.is_open: + messages.warning(request, _("Bill not in open state.")) + else: + if Bill.objects.filter(id__in=bill_ids, is_open=False).exists(): + messages.warning(request, _("Not all bills are in open state.")) + title = _("Manage bill lines of multiple bills") + context = { + 'title': title, + 'bill': bill, + } + context.update(extra_context or {}) + return super().changelist_view(request, context) + + +class BillAdminMixin(AccountAdminMixin): + @mark_safe + def display_total_with_subtotals(self, bill): + if bill.pk: + currency = settings.BILLS_CURRENCY.lower() + subtotals = [] + for tax, subtotal in bill.compute_subtotals().items(): + subtotals.append(_("Subtotal %s%% VAT %s &%s;") % (tax, subtotal[0], currency)) + subtotals.append(_("Taxes %s%% VAT %s &%s;") % (tax, subtotal[1], currency)) + subtotals = '\n'.join(subtotals) + return '%s &%s;' % (subtotals, bill.compute_total(), currency) + display_total_with_subtotals.short_description = _("total") + display_total_with_subtotals.admin_order_field = 'approx_total' + + @mark_safe + def display_payment_state(self, bill): + if bill.pk: + t_opts = bill.transactions.model._meta + if bill.get_type() == bill.PROFORMA: + return '---' + transactions = bill.transactions.all() + if len(transactions) == 1: + args = (transactions[0].pk,) + view = 'admin:%s_%s_change' % (t_opts.app_label, t_opts.model_name) + url = reverse(view, args=args) + else: + url = reverse('admin:%s_%s_changelist' % (t_opts.app_label, t_opts.model_name)) + url += '?bill=%i' % bill.pk + state = bill.get_payment_state_display().upper() + title = '' + if bill.closed_amends: + state = '%s*' % state + title = _("This bill has been amended, this value may not be valid.") + color = PAYMENT_STATE_COLORS.get(bill.payment_state, 'grey') + return '{name}'.format( + url=url, color=color, name=state, title=title) + display_payment_state.short_description = _("Payment") + + def get_queryset(self, request): + qs = super().get_queryset(request) + qs = qs.annotate( + models.Count('lines'), + # FIXME https://code.djangoproject.com/ticket/10060 + approx_total=Coalesce(Sum( + (F('lines__subtotal') + Coalesce('lines__sublines__total', 0)) * (1+F('lines__tax')/100), + ), 0), + ) + qs = qs.prefetch_related( + Prefetch('amends', queryset=Bill.objects.filter(is_open=False), to_attr='closed_amends') + ) + return qs.defer('html') + + +class AmendInline(BillAdminMixin, admin.TabularInline): + model = Bill + fields = ( + 'self_link', 'type', 'display_total_with_subtotals', 'display_payment_state', 'is_open', + 'is_sent' + ) + readonly_fields = fields + verbose_name_plural = _("Amends") + can_delete = False + extra = 0 + + self_link = admin_link('__str__') + + def has_add_permission(self, *args, **kwargs): + return False + + +class BillAdmin(BillAdminMixin, ExtendedModelAdmin): + list_display = ( + 'number', 'type_link', 'account_link', 'closed_on_display', 'updated_on_display', + 'num_lines', 'display_total', 'display_payment_state', 'is_sent' + ) + list_filter = ( + BillTypeListFilter, 'is_open', 'is_sent', TotalListFilter, PaymentStateListFilter, + AmendedListFilter, 'account__is_active', + ) + add_fields = ('account', 'type', 'amend_of', 'is_open', 'due_on', 'comments') + change_list_template = 'admin/bills/bill/change_list.html' + fieldsets = ( + (None, { + 'fields': ['number', 'type', (), 'account_link', 'display_total_with_subtotals', + 'display_payment_state', 'is_sent', 'comments'], + }), + (_("Dates"), { + 'classes': ('collapse',), + 'fields': ('created_on_display', 'closed_on_display', 'updated_on_display', + 'due_on'), + }), + (_("Raw"), { + 'classes': ('collapse',), + 'fields': ('html',), + }), + ) + list_prefetch_related = ('transactions', 'lines__sublines') + search_fields = ('number', 'account__username', 'comments') + change_view_actions = [ + actions.manage_lines, actions.view_bill, actions.download_bills, actions.send_bills, + actions.close_bills, actions.amend_bills, actions.close_send_download_bills, + ] + actions = [ + actions.manage_lines, actions.download_bills, actions.close_bills, actions.send_bills, + actions.amend_bills, actions.bill_report, actions.service_report, + actions.close_send_download_bills, list_accounts, + ] + change_readonly_fields = ('account_link', 'type', 'is_open', 'amend_of_link') + readonly_fields = ( + 'number', 'display_total', 'is_sent', 'display_payment_state', 'created_on_display', + 'closed_on_display', 'updated_on_display', 'display_total_with_subtotals', + ) + date_hierarchy = 'closed_on' + + created_on_display = admin_date('created_on', short_description=_("Created")) + closed_on_display = admin_date('closed_on', short_description=_("Closed")) + updated_on_display = admin_date('updated_on', short_description=_("Updated")) + amend_of_link = admin_link('amend_of') + +# def amend_links(self, bill): +# links = [] +# for amend in bill.amends.all(): +# url = reverse('admin:bills_bill_change', args=(amend.id,)) +# links.append('{num}'.format(url=url, num=amend.number)) +# return '
    '.join(links) +# amend_links.short_description = _("Amends") +# amend_links.allow_tags = True + + def num_lines(self, bill): + return bill.lines__count + num_lines.admin_order_field = 'lines__count' + num_lines.short_description = _("lines") + + def display_total(self, bill): + currency = settings.BILLS_CURRENCY.lower() + return format_html('{} &{};', bill.compute_total(), currency) + display_total.short_description = _("total") + display_total.admin_order_field = 'approx_total' + + def type_link(self, bill): + bill_type = bill.type.lower() + url = reverse('admin:bills_%s_changelist' % bill_type) + return format_html('{}', url, bill.get_type_display()) + type_link.short_description = _("type") + type_link.admin_order_field = 'type' + + def get_urls(self): + """ Hook bill lines management URLs on bill admin """ + urls = super().get_urls() + admin_site = self.admin_site + extra_urls = [ + url("^manage-lines/$", + admin_site.admin_view(BillLineManagerAdmin(BillLine, admin_site).changelist_view), + name='bills_bill_manage_lines'), + ] + return extra_urls + urls + + def get_readonly_fields(self, request, obj=None): + fields = super().get_readonly_fields(request, obj) + if obj and not obj.is_open: + fields += self.add_fields + return fields + + def get_fieldsets(self, request, obj=None): + fieldsets = super().get_fieldsets(request, obj) + if obj: + # Switches between amend_of_link and amend_links fields + fields = fieldsets[0][1]['fields'] + if obj.amend_of_id: + fields[2] = 'amend_of_link' + else: + fields[2] = () + if obj.is_open: + fieldsets = fieldsets[0:-1] + return fieldsets + + def get_change_view_actions(self, obj=None): + actions = super().get_change_view_actions(obj) + exclude = [] + if obj: + if not obj.is_open: + exclude += ['close_bills', 'close_send_download_bills'] + if obj.type not in obj.AMEND_MAP: + exclude += ['amend_bills'] + return [action for action in actions if action.__name__ not in exclude] + + def get_inline_instances(self, request, obj=None): + cls = type(self) + if obj and not obj.is_open: + if obj.amends.all(): + cls.inlines = [AmendInline, ClosedBillLineInline] + else: + cls.inlines = [ClosedBillLineInline] + else: + cls.inlines = [BillLineInline] + return super().get_inline_instances(request, obj) + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'comments': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 4}) + elif db_field.name == 'html': + kwargs['widget'] = forms.Textarea(attrs={'cols': 150, 'rows': 20}) + formfield = super().formfield_for_dbfield(db_field, **kwargs) + if db_field.name == 'amend_of': + formfield.queryset = formfield.queryset.filter(is_open=False) + return formfield + + def change_view(self, request, object_id, **kwargs): + # TODO raise404, here and everywhere + bill = self.get_object(request, unquote(object_id)) + actions.validate_contact(request, bill, error=False) + return super().change_view(request, object_id, **kwargs) + + +admin.site.register(Bill, BillAdmin) +admin.site.register(Invoice, BillAdmin) +admin.site.register(AmendmentInvoice, BillAdmin) +admin.site.register(AbonoInvoice, BillAdmin) +admin.site.register(Fee, BillAdmin) +admin.site.register(AmendmentFee, BillAdmin) +admin.site.register(ProForma, BillAdmin) +admin.site.register(BillLine, BillLineAdmin) + + +class BillContactInline(admin.StackedInline): + model = BillContact + fields = ('name', 'address', ('city', 'zipcode'), 'country', 'vat') + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'name': + kwargs['widget'] = forms.TextInput(attrs={'size':'90'}) + if db_field.name == 'address': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 2}) + if db_field.name == 'email_usage': + kwargs['widget'] = PaddingCheckboxSelectMultiple(45) + return super().formfield_for_dbfield(db_field, **kwargs) + + +def has_bill_contact(account): + return hasattr(account, 'billcontact') +has_bill_contact.boolean = True +has_bill_contact.admin_order_field = 'billcontact' + + +insertattr(AccountAdmin, 'inlines', BillContactInline) +insertattr(AccountAdmin, 'list_display', has_bill_contact) +insertattr(AccountAdmin, 'list_filter', HasBillContactListFilter) +insertattr(AccountAdmin, 'list_select_related', 'billcontact') diff --git a/orchestra/contrib/bills/api.py b/orchestra/contrib/bills/api.py new file mode 100644 index 00000000..7d050b4c --- /dev/null +++ b/orchestra/contrib/bills/api.py @@ -0,0 +1,29 @@ +from django.http import HttpResponse +from rest_framework import viewsets +from rest_framework.decorators import action + +from orchestra.api import router, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin +from orchestra.utils.html import html_to_pdf + +from .models import Bill +from .serializers import BillSerializer + + + +class BillViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = Bill.objects.all() + serializer_class = BillSerializer + + @action(detail=True, methods=['get']) + def document(self, request, pk): + bill = self.get_object() + content_type = request.META.get('HTTP_ACCEPT') + if content_type == 'application/pdf': + pdf = html_to_pdf(bill.html or bill.render()) + return HttpResponse(pdf, content_type='application/pdf') + else: + return HttpResponse(bill.html or bill.render()) + + +router.register('bills', BillViewSet) diff --git a/orchestra/contrib/bills/apps.py b/orchestra/contrib/bills/apps.py new file mode 100644 index 00000000..ecc74588 --- /dev/null +++ b/orchestra/contrib/bills/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + +from orchestra.core import accounts + + +class BillsConfig(AppConfig): + name = 'orchestra.contrib.bills' + verbose_name = 'Bills' + + def ready(self): + from .models import Bill + accounts.register(Bill, icon='invoice.png') diff --git a/orchestra/contrib/bills/filters.py b/orchestra/contrib/bills/filters.py new file mode 100644 index 00000000..adcf5753 --- /dev/null +++ b/orchestra/contrib/bills/filters.py @@ -0,0 +1,160 @@ +from django.contrib.admin import SimpleListFilter +from django.urls import reverse +from django.db.models import Q +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from . models import Bill + + +class BillTypeListFilter(SimpleListFilter): + """ Filter tickets by created_by according to request.user """ + title = 'Type' + parameter_name = '' + + def __init__(self, request, *args, **kwargs): + super(BillTypeListFilter, self).__init__(request, *args, **kwargs) + self.request = request + + def lookups(self, request, model_admin): + return ( + ('bill', _("All")), + ('invoice', _("Invoice")), + ('fee', _("Fee")), + ('proforma', _("Pro-forma")), + ('amendmentfee', _("Amendment fee")), + ('amendmentinvoice', _("Amendment invoice")), + ) + + def queryset(self, request, queryset): + return queryset + + def value(self): + return self.request.path.split('/')[-2] + + def choices(self, cl): + query = self.request.GET.urlencode() + for lookup, title in self.lookup_choices: + yield { + 'selected': self.value() == lookup, + 'query_string': reverse('admin:bills_%s_changelist' % lookup) + '?%s' % query, + 'display': title, + } + + +class TotalListFilter(SimpleListFilter): + title = _("total") + parameter_name = 'total' + + def lookups(self, request, model_admin): + return ( + ('gt', mark_safe("total > 0")), + ('lt', mark_safe("total < 0")), + ('eq', "total = 0"), + ('ne', mark_safe("total ≠ 0")), + ) + + def queryset(self, request, queryset): + if self.value() == 'gt': + return queryset.filter(approx_total__gt=0) + elif self.value() == 'eq': + return queryset.filter(approx_total=0) + elif self.value() == 'lt': + return queryset.filter(approx_total__lt=0) + elif self.value() == 'ne': + return queryset.exclude(approx_total=0) + return queryset + + +class HasBillContactListFilter(SimpleListFilter): + """ Filter Nodes by group according to request.user """ + title = _("has bill contact") + parameter_name = 'bill' + + def lookups(self, request, model_admin): + return ( + ('True', _("Yes")), + ('False', _("No")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(billcontact__isnull=False) + elif self.value() == 'False': + return queryset.filter(billcontact__isnull=True) + + +class PaymentStateListFilter(SimpleListFilter): + title = _("payment state") + parameter_name = 'payment_state' + + def lookups(self, request, model_admin): + return ( + ('OPEN', _("Open")), + ('PAID', _("Paid")), + ('PENDING', _("Pending")), + ('BAD_DEBT', _("Bad debt")), + ) + + def queryset(self, request, queryset): + # FIXME use queryset.computed_total instead of approx_total, bills.admin.BillAdmin.get_queryset + Transaction = queryset.model.transactions.field.remote_field.related_model + if self.value() == 'OPEN': + return queryset.filter(Q(is_open=True)|Q(type=queryset.model.PROFORMA)) + elif self.value() == 'PAID': + zeros = queryset.filter(approx_total=0, approx_total__isnull=True) + zeros = zeros.values_list('id', flat=True) + amounts = Transaction.objects.exclude(bill_id__in=zeros).secured().group_by('bill_id') + paid = [] + relevant = queryset.exclude(approx_total=0, approx_total__isnull=True, is_open=True) + for bill_id, total in relevant.values_list('id', 'approx_total'): + try: + amount = sum([t.amount for t in amounts[bill_id]]) + except KeyError: + pass + else: + if abs(total) <= abs(amount): + paid.append(bill_id) + return queryset.filter( + Q(approx_total=0) | + Q(approx_total__isnull=True) | + Q(id__in=paid) + ).exclude(is_open=True) + elif self.value() == 'PENDING': + has_transaction = queryset.exclude(transactions__isnull=True) + non_rejected = has_transaction.exclude(transactions__state=Transaction.REJECTED) + paid = non_rejected.exclude(transactions__state=Transaction.SECURED) + paid = paid.values_list('id', flat=True).distinct() + return queryset.filter(pk__in=paid) + elif self.value() == 'BAD_DEBT': + closed = queryset.filter(is_open=False).exclude(approx_total=0) + return closed.filter( + Q(transactions__state=Transaction.REJECTED) | + Q(transactions__isnull=True) + ) + + +class AmendedListFilter(SimpleListFilter): + title = _("amended") + parameter_name = 'amended' + + def lookups(self, request, model_admin): + return ( + ('3', _("Closed amends")), + ('2', _("Open amends")), + ('1', _("Any amends")), + ('0', _("No amends")), + ) + + def queryset(self, request, queryset): + if self.value() is None: + return queryset + amended = queryset.filter(amends__isnull=False) + if self.value() == '1': + return amended.distinct() + elif self.value() == '2': + return amended.filter(amends__is_open=True).distinct() + elif self.value() == '3': + return amended.filter(amends__is_open=False).distinct() + elif self.value() == '0': + return queryset.filter(amends__isnull=True).distinct() diff --git a/orchestra/contrib/bills/forms.py b/orchestra/contrib/bills/forms.py new file mode 100644 index 00000000..b4759928 --- /dev/null +++ b/orchestra/contrib/bills/forms.py @@ -0,0 +1,49 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin.utils import admin_link +from orchestra.forms import SpanWidget + + +class SelectSourceForm(forms.ModelForm): + bill_link = forms.CharField(label=_("Number"), required=False, widget=SpanWidget) + account_link = forms.CharField(label=_("Account"), required=False, widget=SpanWidget) + show_total = forms.CharField(label=_("Total"), required=False, widget=SpanWidget) + display_type = forms.CharField(label=_("Type"), required=False, widget=SpanWidget) + source = forms.ChoiceField(label=_("Source"), required=False) + + class Meta: + fields = ( + 'bill_link', 'display_type', 'account_link', 'show_total', 'source' + ) + + def __init__(self, *args, **kwargs): + super(SelectSourceForm, self).__init__(*args, **kwargs) + bill = kwargs.get('instance') + if bill: + total = bill.compute_total() + sources = bill.account.paymentsources.filter(is_active=True) + recharge = bool(total < 0) + choices = [(None, '-----------')] + for source in sources: + if not recharge or source.method_class().allow_recharge: + choices.append((source.pk, str(source))) + self.fields['source'].choices = choices + self.fields['source'].initial = choices[-1][0] + self.fields['show_total'].widget.display = total + self.fields['bill_link'].widget.display = admin_link('__str__')(bill) + self.fields['display_type'].widget.display = bill.get_type_display() + self.fields['account_link'].widget.display = admin_link('account')(bill) + + def clean_source(self): + source_id = self.cleaned_data['source'] + if not source_id: + return None + source_model = self.instance.account.paymentsources.model + return source_model.objects.get(id=source_id) + + def has_changed(self): + return False + + def save(self, commit=True): + pass diff --git a/orchestra/contrib/bills/helpers.py b/orchestra/contrib/bills/helpers.py new file mode 100644 index 00000000..d255e931 --- /dev/null +++ b/orchestra/contrib/bills/helpers.py @@ -0,0 +1,44 @@ +from django.contrib import messages +from django.urls import reverse +from django.utils.encoding import force_str +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.text import capfirst +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin.utils import change_url + + +def validate_contact(request, bill, error=True): + """ checks if all the preconditions for bill generation are met """ + msg = _('{relation} account "{account}" does not have a declared invoice contact. ' + 'You should provide one') + valid = True + send = messages.error if error else messages.warning + if not hasattr(bill.account, 'billcontact'): + account = force_str(bill.account) + url = reverse('admin:accounts_account_change', args=(bill.account_id,)) + message = msg.format(relation=_("Related"), account=account, url=url) + send(request, mark_safe(message)) + valid = False + main = type(bill).account.field.related_model.objects.get_main() + if not hasattr(main, 'billcontact'): + account = force_str(main) + url = reverse('admin:accounts_account_change', args=(main.id,)) + message = msg.format(relation=_("Main"), account=account, url=url) + send(request, mark_safe(message)) + valid = False + return valid + + +def set_context_emails(modeladmin, request, queryset): + opts = modeladmin.model._meta + bills = [] + for bill in queryset: + emails = ', '.join(bill.get_billing_contact_emails()) + bills.append(format_html('{0}: {2} {3}', + capfirst(opts.verbose_name), change_url(bill), bill, emails) + ) + return { + 'display_objects': bills + } diff --git a/orchestra/contrib/bills/locale/ca/LC_MESSAGES/django.mo b/orchestra/contrib/bills/locale/ca/LC_MESSAGES/django.mo new file mode 100644 index 00000000..19d238d4 Binary files /dev/null and b/orchestra/contrib/bills/locale/ca/LC_MESSAGES/django.mo differ diff --git a/orchestra/contrib/bills/locale/ca/LC_MESSAGES/django.po b/orchestra/contrib/bills/locale/ca/LC_MESSAGES/django.po new file mode 100644 index 00000000..541a4546 --- /dev/null +++ b/orchestra/contrib/bills/locale/ca/LC_MESSAGES/django.po @@ -0,0 +1,749 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2019-12-20 11:56+0100\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: actions.py:33 +msgid "View" +msgstr "Vista" + +#: actions.py:45 +msgid "Selected bills should be in open state" +msgstr "Les factures seleccionades han d'estar en estat obert" + +#: actions.py:60 +msgid "Selected bills have been closed" +msgstr "Les factures seleccionades han estat tancades" + +#: actions.py:73 +#, python-format +msgid "One related transaction has been created" +msgstr "S'ha creat una transacció" + +#: actions.py:74 +#, python-format +msgid "%(num)i related transactions have been created" +msgstr "S'han creat les %(num)i següents transaccions" + +#: actions.py:80 +msgid "Are you sure about closing the following bills?" +msgstr "Estàs a punt de tancar les següents factures, estàs segur?" + +#: actions.py:81 +msgid "" +"Once a bill is closed it can not be further modified.

    Please select a " +"payment source for the selected bills" +msgstr "" +"Una vegada la factura estigui tancada no podrà ser modificada.

    Si us " +"plau selecciona un mètode de pagament per les factures seleccionades" + +#: actions.py:97 +msgid "Close" +msgstr "Tanca" + +#: actions.py:115 +msgid "One bill has been sent." +msgstr "S'ha creat una factura" + +#: actions.py:116 +#, python-format +msgid "%i bills have been sent." +msgstr "S'han enviat %i factures." + +#: actions.py:123 +msgid "Resend" +msgstr "Reenviat" + +#: actions.py:146 +msgid "Download" +msgstr "Descarrega" + +#: actions.py:162 +msgid "C.S.D." +msgstr "" + +#: actions.py:164 +msgid "Close, send and download bills in one shot." +msgstr "" + +#: actions.py:225 +#, python-format +msgid "%(norders)s orders and %(nlines)s lines undoed." +msgstr "%(norders)s ordres i %(nlines)s línies desfetes." + +#: actions.py:244 +msgid "Lines moved" +msgstr "Línies mogudes" + +#: actions.py:257 +msgid "Selected bills should be in closed state" +msgstr "Les factures seleccionades han d'estar en estat obert" + +#: actions.py:259 +#, python-format +msgid "%s can not be amended." +msgstr "" + +#: actions.py:279 +#, python-format +msgid "%(type)s of %(related_type)s %(number)s and creation date %(date)s" +msgstr "%(type)s de %(related_type)s %(number)s amb data de creació %(date)s" + +#: actions.py:286 +#, python-format +msgid "%(related_type)s %(number)s subtotal for tax %(tax)s%%" +msgstr "%(related_type)s %(number)s subtotal %(tax)s%%" + +#: actions.py:303 +#, python-format +msgid "One amendment bill have been generated." +msgstr "S'ha creat una transacció" + +#: actions.py:304 +#, python-format +msgid "%(num)i amendment bills have been generated." +msgstr "S'han creat les %(num)i següents transaccions" + +#: actions.py:307 +msgid "Amend" +msgstr "" + +#: admin.py:80 admin.py:126 admin.py:180 forms.py:11 +#: templates/admin/bills/bill/report.html:43 +#: templates/admin/bills/bill/report.html:70 +msgid "Total" +msgstr "Total" + +#: admin.py:112 +msgid "Description" +msgstr "Descripció" + +#: admin.py:120 +msgid "Subtotal" +msgstr "Subtotal" + +#: admin.py:146 +#, fuzzy +#| msgid "Total" +msgid "Totals" +msgstr "Total" + +#: admin.py:150 +msgid "Order" +msgstr "" + +#: admin.py:169 +msgid "Is open" +msgstr "És oberta" + +#: admin.py:175 +#, fuzzy +#| msgid "Subline" +msgid "Sublines" +msgstr "Sublínia" + +#: admin.py:221 +msgid "No bills selected." +msgstr "No hi ha factures seleccionades" + +#: admin.py:229 +#, fuzzy, python-format +#| msgid "Manage %s bill lines." +msgid "Manage %s bill lines" +msgstr "Gestiona %s línies de factura." + +#: admin.py:231 +msgid "Bill not in open state." +msgstr "La factura no està en estat obert" + +#: admin.py:234 +msgid "Not all bills are in open state." +msgstr "No totes les factures estan en estat obert" + +#: admin.py:235 +#, fuzzy +#| msgid "Manage bill lines of multiple bills." +msgid "Manage bill lines of multiple bills" +msgstr "Gestiona línies de factura de multiples factures." + +#: admin.py:250 +#, python-format +msgid "Subtotal %s%% VAT %s &%s;" +msgstr "" + +#: admin.py:251 +#, python-format +msgid "Taxes %s%% VAT %s &%s;" +msgstr "" + +#: admin.py:255 admin.py:381 filters.py:46 +#: templates/bills/microspective.html:123 +msgid "total" +msgstr "total" + +#: admin.py:275 +msgid "This bill has been amended, this value may not be valid." +msgstr "" + +#: admin.py:280 +msgid "Payment" +msgstr "Pagament" + +#: admin.py:304 +#, fuzzy +#| msgid "amended line" +msgid "Amends" +msgstr "línia rectificada" + +#: admin.py:330 +msgid "Dates" +msgstr "" + +#: admin.py:335 +msgid "Raw" +msgstr "Raw" + +#: admin.py:358 models.py:75 +msgid "Created" +msgstr "Creada" + +#: admin.py:359 +#, fuzzy +#| msgid "Close" +msgid "Closed" +msgstr "Tanca" + +#: admin.py:360 +#, fuzzy +#| msgid "updated on" +msgid "Updated" +msgstr "actualitzada el" + +#: admin.py:375 +msgid "lines" +msgstr "línies" + +#: admin.py:389 models.py:108 models.py:501 +msgid "type" +msgstr "tipus" + +#: filters.py:21 +msgid "All" +msgstr "Tot" + +#: filters.py:22 models.py:91 +msgid "Invoice" +msgstr "Factura" + +#: filters.py:23 models.py:93 +msgid "Fee" +msgstr "Quota de soci" + +#: filters.py:24 +msgid "Pro-forma" +msgstr "Pro-forma" + +#: filters.py:25 +msgid "Amendment fee" +msgstr "Rectificació de quota de soci" + +#: filters.py:26 models.py:92 +msgid "Amendment invoice" +msgstr "Factura rectificativa" + +#: filters.py:71 +msgid "has bill contact" +msgstr "té contacte de facturació" + +#: filters.py:76 +msgid "Yes" +msgstr "Si" + +#: filters.py:77 +msgid "No" +msgstr "No" + +#: filters.py:88 +msgid "payment state" +msgstr "Pagament" + +#: filters.py:93 models.py:74 +msgid "Open" +msgstr "" + +#: filters.py:94 models.py:78 +msgid "Paid" +msgstr "Pagat" + +#: filters.py:95 +msgid "Pending" +msgstr "Pendent" + +#: filters.py:96 models.py:81 +msgid "Bad debt" +msgstr "Incobrable" + +#: filters.py:138 +#, fuzzy +#| msgid "amended line" +msgid "amended" +msgstr "línia rectificada" + +#: filters.py:143 +#, fuzzy +#| msgid "Due date" +msgid "Closed amends" +msgstr "Data de pagament" + +#: filters.py:144 +#, fuzzy +#| msgid "Due date" +msgid "Open amends" +msgstr "Data de pagament" + +#: filters.py:145 +#, fuzzy +#| msgid "amended line" +msgid "Any amends" +msgstr "línia rectificada" + +#: filters.py:146 +msgid "No amends" +msgstr "" + +#: forms.py:9 templates/admin/bills/bill/report.html:64 +msgid "Number" +msgstr "Número" + +#: forms.py:10 +msgid "Account" +msgstr "Compte" + +#: forms.py:12 +msgid "Type" +msgstr "Tipus" + +#: forms.py:13 +msgid "Source" +msgstr "Font" + +#: helpers.py:14 +msgid "" +"{relation} account \"{account}\" does not have a declared invoice contact. " +"You should provide one" +msgstr "" +"{relation} compte \"{account}\" no te un contacte de facturació. Hauries de " +"proporcionar un" + +#: helpers.py:21 +msgid "Related" +msgstr "Relacionat" + +#: helpers.py:28 +msgid "Main" +msgstr "Principal" + +#: models.py:26 models.py:104 +msgid "account" +msgstr "compte" + +#: models.py:28 +msgid "name" +msgstr "nom" + +#: models.py:29 +msgid "Account full name will be used when left blank." +msgstr "S'emprarà el nom complet del compte quan es deixi en blanc." + +#: models.py:30 +msgid "address" +msgstr "adreça" + +#: models.py:31 +msgid "city" +msgstr "ciutat" + +#: models.py:33 +msgid "zip code" +msgstr "codi postal" + +#: models.py:34 +msgid "Enter a valid zipcode." +msgstr "Introdueix un codi postal vàlid." + +#: models.py:35 +msgid "country" +msgstr "país" + +#: models.py:38 templates/admin/bills/bill/report.html:65 +msgid "VAT number" +msgstr "NIF" + +#: models.py:76 +msgid "Processed" +msgstr "" + +#: models.py:77 +#, fuzzy +#| msgid "amended line" +msgid "Amended" +msgstr "línia rectificada" + +#: models.py:79 +msgid "Incomplete" +msgstr "" + +#: models.py:80 +msgid "Executed" +msgstr "" + +#: models.py:94 +msgid "Amendment Fee" +msgstr "Rectificació de quota de soci" + +#: models.py:95 +#, fuzzy +#| msgid "Invoice" +msgid "Abono Invoice" +msgstr "Abonament" + +#: models.py:96 +msgid "Pro forma" +msgstr "Pro forma" + +#: models.py:103 +msgid "number" +msgstr "número" + +#: models.py:106 +#, fuzzy +#| msgid "amended line" +msgid "amend of" +msgstr "línia rectificada" + +#: models.py:109 +msgid "created on" +msgstr "creat el" + +#: models.py:110 +msgid "closed on" +msgstr "tancat el" + +#: models.py:111 +msgid "open" +msgstr "obert" + +#: models.py:112 +msgid "sent" +msgstr "enviat" + +#: models.py:113 +msgid "due on" +msgstr "es deu" + +#: models.py:114 +msgid "updated on" +msgstr "actualitzada el" + +#: models.py:116 +msgid "comments" +msgstr "comentaris" + +#: models.py:117 +msgid "HTML" +msgstr "HTML" + +#: models.py:200 +#, python-format +msgid "Type %s is not an amendment." +msgstr "" + +#: models.py:202 +msgid "Amend of related account doesn't match bill account." +msgstr "" + +#: models.py:204 +#, fuzzy +#| msgid "Bill not in open state." +msgid "Related invoice is in open state." +msgstr "La factura no està en estat obert" + +#: models.py:206 +msgid "Related invoice is an amendment." +msgstr "" + +#: models.py:419 +msgid "bill" +msgstr "factura" + +#: models.py:420 models.py:499 templates/bills/microspective.html:75 +msgid "description" +msgstr "descripció" + +#: models.py:421 +msgid "rate" +msgstr "tarifa" + +#: models.py:422 +msgid "quantity" +msgstr "quantitat" + +#: models.py:424 +#, fuzzy +#| msgid "quantity" +msgid "Verbose quantity" +msgstr "quantitat" + +#: models.py:425 templates/admin/bills/bill/report.html:47 +#: templates/bills/microspective.html:79 +#: templates/bills/microspective.html:116 +msgid "subtotal" +msgstr "subtotal" + +#: models.py:426 +msgid "tax" +msgstr "impostos" + +#: models.py:427 +msgid "start" +msgstr "iniciar" + +#: models.py:428 +msgid "end" +msgstr "finalitzar" + +#: models.py:431 +msgid "Informative link back to the order" +msgstr "Enllaç informatiu de l'ordre" + +#: models.py:432 +msgid "order billed" +msgstr "ordre facturada" + +#: models.py:433 +msgid "order billed until" +msgstr "ordre facturada fins a" + +#: models.py:434 +msgid "created" +msgstr "creada" + +#: models.py:436 +msgid "amended line" +msgstr "línia rectificada" + +#: models.py:492 +msgid "Volume" +msgstr "Volum" + +#: models.py:493 +msgid "Compensation" +msgstr "Compensació" + +#: models.py:494 +msgid "Other" +msgstr "Altre" + +#: models.py:498 +msgid "bill line" +msgstr "línia de factura" + +#: templates/admin/bills/bill/change_list.html:9 +#, fuzzy +#| msgid "lines" +msgid "Lines" +msgstr "línies" + +#: templates/admin/bills/bill/change_list.html:15 +#, fuzzy +#| msgid "bill" +msgid "Add bill" +msgstr "factura" + +#: templates/admin/bills/bill/close_send_download_bills.html:57 +msgid "Yes, I'm sure" +msgstr "" + +#: templates/admin/bills/bill/report.html:42 +msgid "Summary" +msgstr "" + +#: templates/admin/bills/bill/report.html:47 +#: templates/admin/bills/bill/report.html:51 +#: templates/admin/bills/bill/report.html:69 +#: templates/bills/microspective.html:116 +#: templates/bills/microspective.html:119 +msgid "VAT" +msgstr "IVA" + +#: templates/admin/bills/bill/report.html:51 +#: templates/bills/microspective.html:119 +msgid "taxes" +msgstr "impostos" + +#: templates/admin/bills/bill/report.html:56 +#: templates/admin/bills/billline/report.html:60 +#: templates/bills/microspective.html:54 +msgid "TOTAL" +msgstr "TOTAL" + +#: templates/admin/bills/bill/report.html:66 +msgid "Contact" +msgstr "" + +#: templates/admin/bills/bill/report.html:67 +#, fuzzy +#| msgid "Due date" +msgid "Close date" +msgstr "Data de pagament" + +#: templates/admin/bills/bill/report.html:68 +msgid "Base" +msgstr "" + +#: templates/admin/bills/billline/change_list.html:6 +msgid "Home" +msgstr "" + +#: templates/admin/bills/billline/change_list.html:8 +msgid "Bills" +msgstr "" + +#: templates/admin/bills/billline/change_list.html:9 +msgid "Multiple bills" +msgstr "" + +#: templates/admin/bills/billline/report.html:42 +msgid "Service" +msgstr "" + +#: templates/admin/bills/billline/report.html:43 +msgid "Active" +msgstr "" + +#: templates/admin/bills/billline/report.html:44 +msgid "Cancelled" +msgstr "" + +#: templates/admin/bills/billline/report.html:45 +msgid "Nominal price" +msgstr "" + +#: templates/admin/bills/billline/report.html:46 +#, fuzzy +#| msgid "quantity" +msgid "Quantity" +msgstr "quantitat" + +#: templates/admin/bills/billline/report.html:47 +msgid "Profit" +msgstr "" + +#: templates/bills/microspective-fee.html:115 +msgid "Due date" +msgstr "Data de pagament" + +#: templates/bills/microspective-fee.html:116 +#, python-format +msgid "On %(bank_account)s" +msgstr "Al %(bank_account)s" + +#: templates/bills/microspective-fee.html:122 +#, python-format +msgid "From %(ini)s to %(end)s" +msgstr "De %(ini)s a %(end)s" + +#: templates/bills/microspective-fee.html:144 +msgid "" +"\n" +"With your membership you are supporting ...\n" +msgstr "" +"\n" +"Amb la teva quota de soci estàs donant suport ...\n" + +#: templates/bills/microspective.html:50 +msgid "DUE DATE" +msgstr "VENCIMENT" + +#: templates/bills/microspective.html:58 +#, python-format +msgid "%(bill_type)s DATE" +msgstr "DATA %(bill_type)s" + +#: templates/bills/microspective.html:76 +msgid "period" +msgstr "període" + +#: templates/bills/microspective.html:77 +msgid "hrs/qty" +msgstr "hrs/qnt" + +#: templates/bills/microspective.html:78 +msgid "rate/price" +msgstr "tarifa/preu" + +#: templates/bills/microspective.html:137 +msgid "COMMENTS" +msgstr "COMENTARIS" + +#: templates/bills/microspective.html:145 +msgid "PAYMENT" +msgstr "PAGAMENT" + +#: templates/bills/microspective.html:149 +#, python-format +msgid "" +"\n" +" You can pay our %(type)s by bank transfer.
    \n" +" Please make sure to state your name and the %(type)s number.\n" +" Our bank account number is
    \n" +" " +msgstr "" +"\n" +"Pots pagar aquesta %(type)s per transferència bancaria.
    Inclou el " +"teu nom i el número de %(type)s. El nostre compte bancari és" + +#: templates/bills/microspective.html:160 +msgid "QUESTIONS" +msgstr "PREGUNTES" + +#: templates/bills/microspective.html:161 +#, python-format +msgid "" +"\n" +" If you have any question about your %(type)s, please\n" +" feel free to write us at %(email)s. We will reply as soon as we " +"get\n" +" your message.\n" +" " +msgstr "" +"\n" +" Si tens algun dubte o pregunta sobre la teva %(type)s, si " +"us plau\n" +" contacta amb nosaltres a %(email)s. Et respondrem el més " +"ràpidament possible.\n" +" " + +#, fuzzy +#~| msgid "closed on" +#~ msgid "No closed amends" +#~ msgstr "tancat el" + +#~ msgid "positive price" +#~ msgstr "preu positiu" diff --git a/orchestra/contrib/bills/locale/es/LC_MESSAGES/django.mo b/orchestra/contrib/bills/locale/es/LC_MESSAGES/django.mo new file mode 100644 index 00000000..135b9643 Binary files /dev/null and b/orchestra/contrib/bills/locale/es/LC_MESSAGES/django.mo differ diff --git a/orchestra/contrib/bills/locale/es/LC_MESSAGES/django.po b/orchestra/contrib/bills/locale/es/LC_MESSAGES/django.po new file mode 100644 index 00000000..dad1e5a2 --- /dev/null +++ b/orchestra/contrib/bills/locale/es/LC_MESSAGES/django.po @@ -0,0 +1,728 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2019-12-20 11:56+0100\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: actions.py:33 +msgid "View" +msgstr "Vista" + +#: actions.py:45 +msgid "Selected bills should be in open state" +msgstr "Las facturas seleccionadas están en estado abierto" + +#: actions.py:60 +msgid "Selected bills have been closed" +msgstr "Las facturas seleccionadas han sido cerradas" + +#: actions.py:73 +#, python-format +msgid "One related transaction has been created" +msgstr "Se ha creado una transacción" + +#: actions.py:74 +#, python-format +msgid "%(num)i related transactions have been created" +msgstr "Se han creado %(num)i transacciones" + +#: actions.py:80 +msgid "Are you sure about closing the following bills?" +msgstr "Estás a punto de cerrar las sigüientes facturas. ¿Estás seguro?" + +#: actions.py:81 +msgid "" +"Once a bill is closed it can not be further modified.

    Please select a " +"payment source for the selected bills" +msgstr "" +"Una vez cerrada la factura ya no se podrá modificar.

    Por favor " +"seleciona un metodo de pago para las facturas seleccionadas" + +#: actions.py:97 +msgid "Close" +msgstr "Cerrar" + +#: actions.py:115 +msgid "One bill has been sent." +msgstr "Se ha enviado una factura" + +#: actions.py:116 +#, python-format +msgid "%i bills have been sent." +msgstr "" + +#: actions.py:123 +msgid "Resend" +msgstr "" + +#: actions.py:146 +msgid "Download" +msgstr "Descarga" + +#: actions.py:162 +msgid "C.S.D." +msgstr "" + +#: actions.py:164 +msgid "Close, send and download bills in one shot." +msgstr "" + +#: actions.py:225 +#, python-format +msgid "%(norders)s orders and %(nlines)s lines undoed." +msgstr "" + +#: actions.py:244 +msgid "Lines moved" +msgstr "" + +#: actions.py:257 +msgid "Selected bills should be in closed state" +msgstr "Las facturas seleccionadas están en estado abierto" + +#: actions.py:259 +#, python-format +msgid "%s can not be amended." +msgstr "" + +#: actions.py:279 +#, python-format +msgid "%(type)s of %(related_type)s %(number)s and creation date %(date)s" +msgstr "%(type)s de %(related_type)s %(number)s con fecha de creación %(date)s" + +#: actions.py:286 +#, python-format +msgid "%(related_type)s %(number)s subtotal for tax %(tax)s%%" +msgstr "%(related_type)s %(number)s subtotal %(tax)s%%" + +#: actions.py:303 +#, python-format +msgid "One amendment bill have been generated." +msgstr "Se ha creado una transacción" + +#: actions.py:304 +#, python-format +msgid "%(num)i amendment bills have been generated." +msgstr "Se han creado %(num)i transacciones" + +#: actions.py:307 +msgid "Amend" +msgstr "" + +#: admin.py:80 admin.py:126 admin.py:180 forms.py:11 +#: templates/admin/bills/bill/report.html:43 +#: templates/admin/bills/bill/report.html:70 +msgid "Total" +msgstr "" + +#: admin.py:112 +msgid "Description" +msgstr "" + +#: admin.py:120 +msgid "Subtotal" +msgstr "" + +#: admin.py:146 +msgid "Totals" +msgstr "" + +#: admin.py:150 +msgid "Order" +msgstr "" + +#: admin.py:169 +msgid "Is open" +msgstr "" + +#: admin.py:175 +msgid "Sublines" +msgstr "" + +#: admin.py:221 +msgid "No bills selected." +msgstr "" + +#: admin.py:229 +#, fuzzy, python-format +#| msgid "bill line" +msgid "Manage %s bill lines" +msgstr "linea de factura" + +#: admin.py:231 +msgid "Bill not in open state." +msgstr "" + +#: admin.py:234 +msgid "Not all bills are in open state." +msgstr "" + +#: admin.py:235 +msgid "Manage bill lines of multiple bills" +msgstr "" + +#: admin.py:250 +#, python-format +msgid "Subtotal %s%% VAT %s &%s;" +msgstr "" + +#: admin.py:251 +#, python-format +msgid "Taxes %s%% VAT %s &%s;" +msgstr "" + +#: admin.py:255 admin.py:381 filters.py:46 +#: templates/bills/microspective.html:123 +msgid "total" +msgstr "" + +#: admin.py:275 +msgid "This bill has been amended, this value may not be valid." +msgstr "" + +#: admin.py:280 +msgid "Payment" +msgstr "Pago" + +#: admin.py:304 +#, fuzzy +#| msgid "Amended" +msgid "Amends" +msgstr "Quota rectificativa" + +#: admin.py:330 +msgid "Dates" +msgstr "" + +#: admin.py:335 +msgid "Raw" +msgstr "" + +#: admin.py:358 models.py:75 +msgid "Created" +msgstr "" + +#: admin.py:359 +#, fuzzy +#| msgid "Close" +msgid "Closed" +msgstr "Cerrar" + +#: admin.py:360 +#, fuzzy +#| msgid "updated on" +msgid "Updated" +msgstr "actualizada en" + +#: admin.py:375 +msgid "lines" +msgstr "" + +#: admin.py:389 models.py:108 models.py:501 +msgid "type" +msgstr "" + +#: filters.py:21 +msgid "All" +msgstr "" + +#: filters.py:22 models.py:91 +msgid "Invoice" +msgstr "Factura" + +#: filters.py:23 models.py:93 +msgid "Fee" +msgstr "Cuota de socio" + +#: filters.py:24 +msgid "Pro-forma" +msgstr "" + +#: filters.py:25 +msgid "Amendment fee" +msgstr "Cuota rectificativa" + +#: filters.py:26 models.py:92 +msgid "Amendment invoice" +msgstr "Factura rectificativa" + +#: filters.py:71 +msgid "has bill contact" +msgstr "" + +#: filters.py:76 +msgid "Yes" +msgstr "" + +#: filters.py:77 +msgid "No" +msgstr "" + +#: filters.py:88 +msgid "payment state" +msgstr "Pago" + +#: filters.py:93 models.py:74 +msgid "Open" +msgstr "" + +#: filters.py:94 models.py:78 +msgid "Paid" +msgstr "" + +#: filters.py:95 +msgid "Pending" +msgstr "" + +#: filters.py:96 models.py:81 +msgid "Bad debt" +msgstr "" + +#: filters.py:138 +#, fuzzy +#| msgid "Amended" +msgid "amended" +msgstr "Quota rectificativa" + +#: filters.py:143 +#, fuzzy +#| msgid "Due date" +msgid "Closed amends" +msgstr "Fecha de pago" + +#: filters.py:144 +#, fuzzy +#| msgid "Due date" +msgid "Open amends" +msgstr "Fecha de pago" + +#: filters.py:145 +#, fuzzy +#| msgid "Amended" +msgid "Any amends" +msgstr "Quota rectificativa" + +#: filters.py:146 +msgid "No amends" +msgstr "" + +#: forms.py:9 templates/admin/bills/bill/report.html:64 +msgid "Number" +msgstr "" + +#: forms.py:10 +msgid "Account" +msgstr "" + +#: forms.py:12 +msgid "Type" +msgstr "" + +#: forms.py:13 +msgid "Source" +msgstr "" + +#: helpers.py:14 +msgid "" +"{relation} account \"{account}\" does not have a declared invoice contact. " +"You should provide one" +msgstr "" + +#: helpers.py:21 +msgid "Related" +msgstr "" + +#: helpers.py:28 +msgid "Main" +msgstr "" + +#: models.py:26 models.py:104 +msgid "account" +msgstr "" + +#: models.py:28 +msgid "name" +msgstr "" + +#: models.py:29 +msgid "Account full name will be used when left blank." +msgstr "" + +#: models.py:30 +msgid "address" +msgstr "" + +#: models.py:31 +msgid "city" +msgstr "" + +#: models.py:33 +msgid "zip code" +msgstr "" + +#: models.py:34 +msgid "Enter a valid zipcode." +msgstr "" + +#: models.py:35 +msgid "country" +msgstr "" + +#: models.py:38 templates/admin/bills/bill/report.html:65 +msgid "VAT number" +msgstr "" + +#: models.py:76 +msgid "Processed" +msgstr "" + +#: models.py:77 +msgid "Amended" +msgstr "Quota rectificativa" + +#: models.py:79 +msgid "Incomplete" +msgstr "" + +#: models.py:80 +msgid "Executed" +msgstr "" + +#: models.py:94 +msgid "Amendment Fee" +msgstr "" + +#: models.py:95 +#, fuzzy +#| msgid "Invoice" +msgid "Abono Invoice" +msgstr "Abono" + +#: models.py:96 +msgid "Pro forma" +msgstr "" + +#: models.py:103 +msgid "number" +msgstr "número" + +#: models.py:106 +msgid "amend of" +msgstr "rectificación de" + +#: models.py:109 +msgid "created on" +msgstr "creado en" + +#: models.py:110 +msgid "closed on" +msgstr "cerrada en" + +#: models.py:111 +msgid "open" +msgstr "abierta" + +#: models.py:112 +msgid "sent" +msgstr "enviada" + +#: models.py:113 +msgid "due on" +msgstr "vencimiento" + +#: models.py:114 +msgid "updated on" +msgstr "actualizada en" + +#: models.py:116 +msgid "comments" +msgstr "comentarios" + +#: models.py:117 +msgid "HTML" +msgstr "HTML" + +#: models.py:200 +#, python-format +msgid "Type %s is not an amendment." +msgstr "" + +#: models.py:202 +msgid "Amend of related account doesn't match bill account." +msgstr "" + +#: models.py:204 +#, fuzzy +#| msgid "Selected bills should be in open state" +msgid "Related invoice is in open state." +msgstr "Las facturas seleccionadas están en estado abierto" + +#: models.py:206 +msgid "Related invoice is an amendment." +msgstr "" + +#: models.py:419 +msgid "bill" +msgstr "factura" + +#: models.py:420 models.py:499 templates/bills/microspective.html:75 +msgid "description" +msgstr "descripción" + +#: models.py:421 +msgid "rate" +msgstr "tarifa" + +#: models.py:422 +msgid "quantity" +msgstr "cantidad" + +#: models.py:424 +msgid "Verbose quantity" +msgstr "Cantidad" + +#: models.py:425 templates/admin/bills/bill/report.html:47 +#: templates/bills/microspective.html:79 +#: templates/bills/microspective.html:116 +msgid "subtotal" +msgstr "subtotal" + +#: models.py:426 +msgid "tax" +msgstr "impuesto" + +#: models.py:427 +msgid "start" +msgstr "inicio" + +#: models.py:428 +msgid "end" +msgstr "fín" + +#: models.py:431 +msgid "Informative link back to the order" +msgstr "" + +#: models.py:432 +msgid "order billed" +msgstr "" + +#: models.py:433 +msgid "order billed until" +msgstr "" + +#: models.py:434 +msgid "created" +msgstr "creado" + +#: models.py:436 +msgid "amended line" +msgstr "linea rectificativa" + +#: models.py:492 +msgid "Volume" +msgstr "Volumen" + +#: models.py:493 +msgid "Compensation" +msgstr "Compensación" + +#: models.py:494 +msgid "Other" +msgstr "Otro" + +#: models.py:498 +msgid "bill line" +msgstr "linea de factura" + +#: templates/admin/bills/bill/change_list.html:9 +msgid "Lines" +msgstr "" + +#: templates/admin/bills/bill/change_list.html:15 +#, fuzzy +#| msgid "bill" +msgid "Add bill" +msgstr "factura" + +#: templates/admin/bills/bill/close_send_download_bills.html:57 +msgid "Yes, I'm sure" +msgstr "" + +#: templates/admin/bills/bill/report.html:42 +msgid "Summary" +msgstr "" + +#: templates/admin/bills/bill/report.html:47 +#: templates/admin/bills/bill/report.html:51 +#: templates/admin/bills/bill/report.html:69 +#: templates/bills/microspective.html:116 +#: templates/bills/microspective.html:119 +msgid "VAT" +msgstr "IVA" + +#: templates/admin/bills/bill/report.html:51 +#: templates/bills/microspective.html:119 +msgid "taxes" +msgstr "impuestos" + +#: templates/admin/bills/bill/report.html:56 +#: templates/admin/bills/billline/report.html:60 +#: templates/bills/microspective.html:54 +msgid "TOTAL" +msgstr "TOTAL" + +#: templates/admin/bills/bill/report.html:66 +msgid "Contact" +msgstr "Contacto" + +#: templates/admin/bills/bill/report.html:67 +#, fuzzy +#| msgid "Due date" +msgid "Close date" +msgstr "Fecha de pago" + +#: templates/admin/bills/bill/report.html:68 +msgid "Base" +msgstr "Base" + +#: templates/admin/bills/billline/change_list.html:6 +msgid "Home" +msgstr "" + +#: templates/admin/bills/billline/change_list.html:8 +msgid "Bills" +msgstr "" + +#: templates/admin/bills/billline/change_list.html:9 +msgid "Multiple bills" +msgstr "" + +#: templates/admin/bills/billline/report.html:42 +msgid "Service" +msgstr "" + +#: templates/admin/bills/billline/report.html:43 +msgid "Active" +msgstr "" + +#: templates/admin/bills/billline/report.html:44 +msgid "Cancelled" +msgstr "" + +#: templates/admin/bills/billline/report.html:45 +msgid "Nominal price" +msgstr "" + +#: templates/admin/bills/billline/report.html:46 +#, fuzzy +#| msgid "quantity" +msgid "Quantity" +msgstr "cantidad" + +#: templates/admin/bills/billline/report.html:47 +msgid "Profit" +msgstr "" + +#: templates/bills/microspective-fee.html:115 +msgid "Due date" +msgstr "Fecha de pago" + +#: templates/bills/microspective-fee.html:116 +#, python-format +msgid "On %(bank_account)s" +msgstr "En %(bank_account)s" + +#: templates/bills/microspective-fee.html:122 +#, python-format +msgid "From %(ini)s to %(end)s" +msgstr "Desde %(ini)s hasta %(end)s" + +#: templates/bills/microspective-fee.html:144 +msgid "" +"\n" +"With your membership you are supporting ...\n" +msgstr "" + +#: templates/bills/microspective.html:50 +msgid "DUE DATE" +msgstr "VENCIMIENTO" + +#: templates/bills/microspective.html:58 +#, python-format +msgid "%(bill_type)s DATE" +msgstr "FECHA %(bill_type)s" + +#: templates/bills/microspective.html:76 +msgid "period" +msgstr "periodo" + +#: templates/bills/microspective.html:77 +msgid "hrs/qty" +msgstr "hrs/cant" + +#: templates/bills/microspective.html:78 +msgid "rate/price" +msgstr "tarifa/precio" + +#: templates/bills/microspective.html:137 +msgid "COMMENTS" +msgstr "COMENTARIOS" + +#: templates/bills/microspective.html:145 +msgid "PAYMENT" +msgstr "PAGO" + +#: templates/bills/microspective.html:149 +#, python-format +msgid "" +"\n" +" You can pay our %(type)s by bank transfer.
    \n" +" Please make sure to state your name and the %(type)s number.\n" +" Our bank account number is
    \n" +" " +msgstr "" +"\n" +"Puedes pagar esta %(type)s por transferencia bancaria.
    Incluye tu " +"nombre y el número de %(type)s. Nuestra cuenta bancaria es" + +#: templates/bills/microspective.html:160 +msgid "QUESTIONS" +msgstr "PREGUNTAS" + +#: templates/bills/microspective.html:161 +#, python-format +msgid "" +"\n" +" If you have any question about your %(type)s, please\n" +" feel free to write us at %(email)s. We will reply as soon as we " +"get\n" +" your message.\n" +" " +msgstr "" +"\n" +" Si tienes alguna duda o pregunta sobre tu %(type)s, por " +"favor\n" +" contacta con nosotros en %(email)s. Te responderemos lo más " +"rapidamente posible.\n" +" " + +#, fuzzy +#~| msgid "closed on" +#~ msgid "No closed amends" +#~ msgstr "cerrada en" diff --git a/orchestra/contrib/bills/models.py b/orchestra/contrib/bills/models.py new file mode 100644 index 00000000..9510f6b6 --- /dev/null +++ b/orchestra/contrib/bills/models.py @@ -0,0 +1,504 @@ +import datetime +from dateutil.relativedelta import relativedelta + +from django.urls import reverse +from django.core.validators import ValidationError, RegexValidator +from django.db import models +from django.db.models import F, Sum +from django.db.models.functions import Coalesce +from django.template import loader +from django.utils import timezone, translation +from django.utils.encoding import force_str +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin.utils import change_url +from orchestra.contrib.accounts.models import Account +from orchestra.contrib.contacts.models import Contact +from orchestra.core import validators +from orchestra.utils.functional import cached +from orchestra.utils.html import html_to_pdf + +from . import settings + + +class BillContact(models.Model): + account = models.OneToOneField('accounts.Account', verbose_name=_("account"), + related_name='billcontact', on_delete=models.CASCADE) + name = models.CharField(_("name"), max_length=256, blank=True, + help_text=_("Account full name will be used when left blank.")) + address = models.TextField(_("address")) + city = models.CharField(_("city"), max_length=128, + default=settings.BILLS_CONTACT_DEFAULT_CITY) + zipcode = models.CharField(_("zip code"), max_length=10, + validators=[RegexValidator(r'^[0-9A-Z]{3,10}$', _("Enter a valid zipcode."))]) + country = models.CharField(_("country"), max_length=20, + choices=settings.BILLS_CONTACT_COUNTRIES, + default=settings.BILLS_CONTACT_DEFAULT_COUNTRY) + vat = models.CharField(_("VAT number"), max_length=64) + + def __str__(self): + return self.name + + def get_name(self): + return self.name or self.account.get_full_name() + + def clean(self): + self.vat = self.vat.strip() + self.city = self.city.strip() + validators.all_valid({ + 'vat': (validators.validate_vat, self.vat, self.country), + 'zipcode': (validators.validate_zipcode, self.zipcode, self.country) + }) + + +class BillManager(models.Manager): + def get_queryset(self): + queryset = super(BillManager, self).get_queryset() + if self.model != Bill: + bill_type = self.model.get_class_type() + queryset = queryset.filter(type=bill_type) + return queryset + + +class Bill(models.Model): + OPEN = '' + CREATED = 'CREATED' + PROCESSED = 'PROCESSED' + AMENDED = 'AMENDED' + PAID = 'PAID' + EXECUTED = 'EXECUTED' + BAD_DEBT = 'BAD_DEBT' + INCOMPLETE = 'INCOMPLETE' + PAYMENT_STATES = ( + (OPEN, _("Open")), + (CREATED, _("Created")), + (PROCESSED, _("Processed")), + (AMENDED, _("Amended")), + (PAID, _("Paid")), + (INCOMPLETE, _('Incomplete')), + (EXECUTED, _("Executed")), + (BAD_DEBT, _("Bad debt")), + ) + BILL = 'BILL' + INVOICE = 'INVOICE' + AMENDMENTINVOICE = 'AMENDMENTINVOICE' + FEE = 'FEE' + AMENDMENTFEE = 'AMENDMENTFEE' + PROFORMA = 'PROFORMA' + ABONOINVOICE = 'ABONOINVOICE' + TYPES = ( + (INVOICE, _("Invoice")), + (AMENDMENTINVOICE, _("Amendment invoice")), + (FEE, _("Fee")), + (AMENDMENTFEE, _("Amendment Fee")), + (ABONOINVOICE, _("Abono Invoice")), + (PROFORMA, _("Pro forma")), + ) + AMEND_MAP = { + INVOICE: AMENDMENTINVOICE, + FEE: AMENDMENTFEE, + } + + number = models.CharField(_("number"), max_length=16, unique=True, blank=True) + account = models.ForeignKey('accounts.Account', verbose_name=_("account"), + related_name='%(class)s', on_delete=models.CASCADE) + amend_of = models.ForeignKey('self', null=True, blank=True, verbose_name=_("amend of"), + related_name='amends', on_delete=models.SET_NULL) + type = models.CharField(_("type"), max_length=16, choices=TYPES) + created_on = models.DateField(_("created on"), auto_now_add=True) + closed_on = models.DateField(_("closed on"), blank=True, null=True, db_index=True) + is_open = models.BooleanField(_("open"), default=True) + is_sent = models.BooleanField(_("sent"), default=False) + due_on = models.DateField(_("due on"), null=True, blank=True) + updated_on = models.DateField(_("updated on"), auto_now=True) +# total = models.DecimalField(max_digits=12, decimal_places=2, null=True) + comments = models.TextField(_("comments"), blank=True) + html = models.TextField(_("HTML"), blank=True) + + objects = BillManager() + + class Meta: + get_latest_by = 'id' + + def __str__(self): + return self.number + + @classmethod + def get_class_type(cls): + if cls is models.DEFERRED: + cls = cls.__base__ + return cls.__name__.upper() + + @cached_property + def total(self): + return self.compute_total() + + @cached_property + def seller(self): + return Account.objects.get_main().billcontact + + @cached_property + def buyer(self): + return self.account.billcontact + + @property + def has_multiple_pages(self): + return self.type != self.FEE + + @cached_property + def payment_state(self): + if self.is_open or self.get_type() == self.PROFORMA: + return self.OPEN + secured = 0 + pending = 0 + created = False + processed = False + executed = False + rejected = False + for transaction in self.transactions.all(): + if transaction.state == transaction.SECURED: + secured += transaction.amount + pending += transaction.amount + elif transaction.state == transaction.WAITTING_PROCESSING: + pending += transaction.amount + created = True + elif transaction.state == transaction.WAITTING_EXECUTION: + pending += transaction.amount + processed = True + elif transaction.state == transaction.EXECUTED: + pending += transaction.amount + executed = True + elif transaction.state == transaction.REJECTED: + rejected = True + else: + raise TypeError("Unknown state") + ongoing = bool(secured != 0 or created or processed or executed) + total = self.compute_total() + if total >= 0: + if secured >= total: + return self.PAID + elif ongoing and pending < total: + return self.INCOMPLETE + else: + if secured <= total: + return self.PAID + elif ongoing and pending > total: + return self.INCOMPLETE + if created: + return self.CREATED + elif processed: + return self.PROCESSED + elif executed: + return self.EXECUTED + return self.BAD_DEBT + + def clean(self): + if self.amend_of_id: + errors = {} + if self.type not in self.AMEND_MAP.values(): + errors['amend_of'] = _("Type %s is not an amendment.") % self.get_type_display() + if self.amend_of.account_id != self.account_id: + errors['account'] = _("Amend of related account doesn't match bill account.") + if self.amend_of.is_open: + errors['amend_of'] = _("Related invoice is in open state.") + if self.amend_of.type in self.AMEND_MAP.values(): + errors['amend_of'] = _("Related invoice is an amendment.") + if errors: + raise ValidationError(errors) + + def get_payment_state_display(self): + value = self.payment_state + return force_str(dict(self.PAYMENT_STATES).get(value, value)) + + def get_current_transaction(self): + return self.transactions.exclude_rejected().first() + + def get_type(self): + return self.type or self.get_class_type() + + @property + def is_amend(self): + return self.type in self.AMEND_MAP.values() + + def get_amend_type(self): + amend_type = self.AMEND_MAP.get(self.type) + if amend_type is None: + raise TypeError("%s has no associated amend type." % self.type) + return amend_type + + def get_number(self): + cls = type(self) + if cls is models.DEFERRED: + cls = cls.__base__ + bill_type = self.get_type() + if bill_type == self.BILL: + raise TypeError('This method can not be used on BILL instances') + bill_type = bill_type.replace('AMENDMENT', 'AMENDMENT_') + prefix = getattr(settings, 'BILLS_%s_NUMBER_PREFIX' % bill_type) + if self.is_open: + prefix = 'O{}'.format(prefix) + year = timezone.now().strftime("%Y") + bills = cls.objects.filter(number__regex=r'^%s%s[0-9]+' % (prefix, year)) + last_number = bills.order_by('-number').values_list('number', flat=True).first() + if last_number is None: + last_number = 0 + else: + last_number = int(last_number[len(prefix)+4:]) + number = last_number + 1 + number_length = settings.BILLS_NUMBER_LENGTH + zeros = (number_length - len(str(number))) * '0' + number = zeros + str(number) + return '{prefix}{year}{number}'.format(prefix=prefix, year=year, number=number) + + def get_due_date(self, payment=None): + now = timezone.now() + if payment: + return now + payment.get_due_delta() + return now + relativedelta(months=1) + + def get_absolute_url(self): + return reverse('admin:bills_bill_view', args=(self.pk,)) + + def close(self, payment=False): + if not self.is_open: + raise TypeError("Bill not in Open state.") + if payment is False: + payment = self.account.paymentsources.get_default() + if not self.due_on: + self.due_on = self.get_due_date(payment=payment) + total = self.compute_total() + transaction = None + if self.get_type() != self.PROFORMA: + transaction = self.transactions.create(bill=self, source=payment, amount=total) + self.closed_on = timezone.now() + self.is_open = False + self.is_sent = False + self.number = self.get_number() + self.html = self.render(payment=payment) + self.save() + return transaction + + def get_billing_contact_emails(self): + return self.account.get_contacts_emails(usages=(Contact.BILLING,)) + + def send(self): + pdf = self.as_pdf() + self.account.send_email( + template=settings.BILLS_EMAIL_NOTIFICATION_TEMPLATE, + context={ + 'bill': self, + 'settings': settings, + }, + email_from=settings.BILLS_SELLER_EMAIL, + usages=(Contact.BILLING,), + attachments=[ + ('%s.pdf' % self.number, pdf, 'application/pdf') + ] + ) + self.is_sent = True + self.save(update_fields=['is_sent']) + + def render(self, payment=False, language=None): + with translation.override(language or self.account.language): + if payment is False: + payment = self.account.paymentsources.get_default() + context = { + 'bill': self, + 'lines': self.lines.all().prefetch_related('sublines'), + 'seller': self.seller, + 'buyer': self.buyer, + 'seller_info': { + 'phone': settings.BILLS_SELLER_PHONE, + 'website': settings.BILLS_SELLER_WEBSITE, + 'email': settings.BILLS_SELLER_EMAIL, + 'bank_account': settings.BILLS_SELLER_BANK_ACCOUNT, + }, + 'currency': settings.BILLS_CURRENCY, + 'payment': payment and payment.get_bill_context(), + 'default_due_date': self.get_due_date(payment=payment), + 'now': timezone.now(), + } + template_name = 'BILLS_%s_TEMPLATE' % self.get_type() + template = getattr(settings, template_name, settings.BILLS_DEFAULT_TEMPLATE) + bill_template = loader.get_template(template) + html = bill_template.render(context) + html = html.replace('-pageskip-', '') + return html + + def as_pdf(self): + html = self.html or self.render() + return html_to_pdf(html, pagination=self.has_multiple_pages) + + def updated(self): + self.updated_on = timezone.now() + self.save(update_fields=('updated_on',)) + + def save(self, *args, **kwargs): + if not self.type: + self.type = self.get_type() + if not self.number: + self.number = self.get_number() + super(Bill, self).save(*args, **kwargs) + + @cached + def compute_subtotals(self): + subtotals = {} + lines = self.lines.annotate(totals=F('subtotal') + Sum(Coalesce('sublines__total', 0))) + for tax, total in lines.values_list('tax', 'totals'): + try: + subtotals[tax] += total + except KeyError: + subtotals[tax] = total + result = {} + for tax, subtotal in subtotals.items(): + result[tax] = [subtotal, round(tax/100*subtotal, 2)] + return result + + @cached + def compute_base(self): + bases = self.lines.annotate( + bases=F('subtotal') + Sum(Coalesce('sublines__total', 0)) + ) + return round(bases.aggregate(Sum('bases'))['bases__sum'] or 0, 2) + + @cached + def compute_tax(self): + taxes = self.lines.annotate( + taxes=(F('subtotal') + Coalesce(Sum('sublines__total'), 0)) * (F('tax')/100) + ) + return round(taxes.aggregate(Sum('taxes'))['taxes__sum'] or 0, 2) + + @cached + def compute_total(self): + if 'lines' in getattr(self, '_prefetched_objects_cache', ()): + total = 0 + for line in self.lines.all(): + line_total = line.compute_total() + total += line_total * (1+line.tax/100) + return round(total, 2) + else: + totals = self.lines.annotate( + totals=(F('subtotal') + Sum(Coalesce('sublines__total', 0))) * (1+F('tax')/100) + ) + return round(totals.aggregate(Sum('totals'))['totals__sum'] or 0, 2) + + +class Invoice(Bill): + class Meta: + proxy = True + + +class AmendmentInvoice(Bill): + class Meta: + proxy = True + + +class AbonoInvoice(Bill): + class Meta: + proxy = True + + +class Fee(Bill): + class Meta: + proxy = True + + +class AmendmentFee(Bill): + class Meta: + proxy = True + + +class ProForma(Bill): + class Meta: + proxy = True + + +class BillLine(models.Model): + """ Base model for bill item representation """ + bill = models.ForeignKey(Bill, verbose_name=_("bill"), related_name='lines', on_delete=models.CASCADE) + description = models.CharField(_("description"), max_length=256) + rate = models.DecimalField(_("rate"), blank=True, null=True, max_digits=12, decimal_places=2) + quantity = models.DecimalField(_("quantity"), blank=True, null=True, max_digits=12, + decimal_places=2) + verbose_quantity = models.CharField(_("Verbose quantity"), max_length=16, blank=True) + subtotal = models.DecimalField(_("subtotal"), max_digits=12, decimal_places=2) + tax = models.DecimalField(_("tax"), max_digits=4, decimal_places=2) + start_on = models.DateField(_("start")) + end_on = models.DateField(_("end"), null=True, blank=True) + order = models.ForeignKey(settings.BILLS_ORDER_MODEL, null=True, blank=True, + related_name='lines', on_delete=models.SET_NULL, + help_text=_("Informative link back to the order")) + order_billed_on = models.DateField(_("order billed"), null=True, blank=True) + order_billed_until = models.DateField(_("order billed until"), null=True, blank=True) + created_on = models.DateField(_("created"), auto_now_add=True) + # Amendment + amended_line = models.ForeignKey('self', verbose_name=_("amended line"), + related_name='amendment_lines', null=True, blank=True, on_delete=models.CASCADE) + + class Meta: + get_latest_by = 'id' + + def __str__(self): + return "#%i" % self.pk if self.pk else self.description + + def get_verbose_quantity(self): + return self.verbose_quantity or self.quantity + + def clean(self): + if not self.verbose_quantity: + quantity = str(self.quantity) + # Strip trailing zeros + if quantity.endswith('0'): + self.verbose_quantity = quantity.strip('0').strip('.') + + def get_verbose_period(self): + from django.template.defaultfilters import date + date_format = "N 'y" + if self.start_on.day != 1 or (self.end_on and self.end_on.day != 1): + date_format = "N j, 'y" + end = date(self.end_on, date_format) + elif self.end_on: + end = date((self.end_on - datetime.timedelta(days=1)), date_format) + ini = date(self.start_on, date_format).capitalize() + if not self.end_on: + return ini + end = end.capitalize() + if ini == end: + return ini + return "{ini} / {end}".format(ini=ini, end=end) + + @cached + def compute_total(self): + total = self.subtotal or 0 + if hasattr(self, 'subline_total'): + total += self.subline_total or 0 + elif 'sublines' in getattr(self, '_prefetched_objects_cache', ()): + total += sum(subline.total for subline in self.sublines.all()) + else: + total += self.sublines.aggregate(sub_total=Sum('total'))['sub_total'] or 0 + return round(total, 2) + + def get_absolute_url(self): + return change_url(self) + + +class BillSubline(models.Model): + """ Subline used for describing an item discount """ + VOLUME = 'VOLUME' + COMPENSATION = 'COMPENSATION' + OTHER = 'OTHER' + TYPES = ( + (VOLUME, _("Volume")), + (COMPENSATION, _("Compensation")), + (OTHER, _("Other")), + ) + + # TODO: order info for undoing + line = models.ForeignKey(BillLine, verbose_name=_("bill line"), related_name='sublines', on_delete=models.CASCADE) + description = models.CharField(_("description"), max_length=256) + total = models.DecimalField(max_digits=12, decimal_places=2) + type = models.CharField(_("type"), max_length=16, choices=TYPES, default=OTHER) + + def __str__(self): + return "%s %i" % (self.description, self.total) diff --git a/orchestra/contrib/bills/serializers.py b/orchestra/contrib/bills/serializers.py new file mode 100644 index 00000000..f73380b3 --- /dev/null +++ b/orchestra/contrib/bills/serializers.py @@ -0,0 +1,34 @@ +from rest_framework import serializers + +from orchestra.api import router +from orchestra.contrib.accounts.models import Account +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .models import Bill, BillLine, BillContact + + +class BillLineSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = BillLine + + + +class BillSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): +# lines = BillLineSerializer(source='lines') + + class Meta: + model = Bill + fields = ( + 'url', 'id', 'number', 'type', 'total', 'is_sent', 'created_on', 'due_on', + 'comments', +# 'lines' + ) + + +class BillContactSerializer(AccountSerializerMixin, serializers.ModelSerializer): + class Meta: + model = BillContact + fields = ('name', 'address', 'city', 'zipcode', 'country', 'vat') + + +router.insert(Account, 'billcontact', BillContactSerializer, required=False) diff --git a/orchestra/contrib/bills/settings.py b/orchestra/contrib/bills/settings.py new file mode 100644 index 00000000..93e15da7 --- /dev/null +++ b/orchestra/contrib/bills/settings.py @@ -0,0 +1,106 @@ +from django_countries import data + +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN + + +BILLS_NUMBER_LENGTH = Setting('BILLS_NUMBER_LENGTH', + 4 +) + + +BILLS_INVOICE_NUMBER_PREFIX = Setting('BILLS_INVOICE_NUMBER_PREFIX', + 'I' +) + + +BILLS_AMENDMENT_INVOICE_NUMBER_PREFIX = Setting('BILLS_AMENDMENT_INVOICE_NUMBER_PREFIX', + 'A' +) + +BILLS_ABONOINVOICE_NUMBER_PREFIX = Setting('BILLS_ABONOINVOICE_NUMBER_PREFIX', + 'AB' +) + +BILLS_FEE_NUMBER_PREFIX = Setting('BILLS_FEE_NUMBER_PREFIX', + 'F' +) + +BILLS_AMENDMENT_FEE_NUMBER_PREFIX = Setting('BILLS_AMENDMENT_FEE_NUMBER_PREFIX', + 'B' +) + + +BILLS_PROFORMA_NUMBER_PREFIX = Setting('BILLS_PROFORMA_NUMBER_PREFIX', + 'P' +) + + +BILLS_DEFAULT_TEMPLATE = Setting('BILLS_DEFAULT_TEMPLATE', + 'bills/microspective.html' +) + + +BILLS_FEE_TEMPLATE = Setting('BILLS_FEE_TEMPLATE', + 'bills/microspective-fee.html' +) + + +BILLS_PROFORMA_TEMPLATE = Setting('BILLS_PROFORMA_TEMPLATE', + 'bills/microspective-proforma.html' +) + + +BILLS_CURRENCY = Setting('BILLS_CURRENCY', + 'euro' +) + + +BILLS_SELLER_PHONE = Setting('BILLS_SELLER_PHONE', + '111-112-11-222' +) + + +BILLS_SELLER_EMAIL = Setting('BILLS_SELLER_EMAIL', + 'sales@{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + + +BILLS_SELLER_WEBSITE = Setting('BILLS_SELLER_WEBSITE', + 'www.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + + +BILLS_SELLER_BANK_ACCOUNT = Setting('BILLS_SELLER_BANK_ACCOUNT', + '0000 0000 00 00000000 (Orchestra Bank)' +) + + +BILLS_EMAIL_NOTIFICATION_TEMPLATE = Setting('BILLS_EMAIL_NOTIFICATION_TEMPLATE', + 'bills/bill-notification.email' +) + + +BILLS_ORDER_MODEL = Setting('BILLS_ORDER_MODEL', + 'orders.Order', + validators=[Setting.validate_model_label] +) + + +BILLS_CONTACT_DEFAULT_CITY = Setting('BILLS_CONTACT_DEFAULT_CITY', + 'Barcelona' +) + + +BILLS_CONTACT_COUNTRIES = Setting('BILLS_CONTACT_COUNTRIES', + tuple((k,v) for k,v in data.COUNTRIES.items()), + serializable=False +) + + +BILLS_CONTACT_DEFAULT_COUNTRY = Setting('BILLS_CONTACT_DEFAULT_COUNTRY', + 'ES', + choices=BILLS_CONTACT_COUNTRIES +) diff --git a/orchestra/contrib/bills/templates/admin/bills/bill/change_list.html b/orchestra/contrib/bills/templates/admin/bills/bill/change_list.html new file mode 100644 index 00000000..220dbfe3 --- /dev/null +++ b/orchestra/contrib/bills/templates/admin/bills/bill/change_list.html @@ -0,0 +1,18 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls %} + + +{% block object-tools-items %} +

  • + {% url 'admin:bills_billline_changelist' as list_url %} + + {% trans "Lines" %} + +
    • +
  • + {% url 'admin:bills_bill_add' as add_url %} + + {% trans "Add bill" %} + +
    • +{% endblock %} diff --git a/orchestra/contrib/bills/templates/admin/bills/bill/close_send_download_bills.html b/orchestra/contrib/bills/templates/admin/bills/bill/close_send_download_bills.html new file mode 100644 index 00000000..d56dbe0e --- /dev/null +++ b/orchestra/contrib/bills/templates/admin/bills/bill/close_send_download_bills.html @@ -0,0 +1,60 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} +{% load i18n l10n %} +{% load admin_urls static utils %} + +{% block extrastyle %} +{{ block.super }} + +{% endblock %} + +{% block content %} +
    +
    +
    +

    {{ content_message | safe }}

    +
      {{ display_objects | unordered_list }}
    +
    +
    {% csrf_token %} + {% block form %} + {% if form %} +
    + {{ form.non_field_errors }} + {% for field in form %} +
    +
    + {{ field.errors }} + {% if field|is_checkbox %} + {{ field }} + {% else %} + {{ field.label_tag }} {{ field }} + {% endif %} +

    {{ field.help_text|safe }}

    +
    +
    + {% endfor %} +
    + {% endif %} + {% endblock %} + {% block formset %} + {% if formset %} + {{ formset.as_admin }} + {% endif %} + {% endblock %} +
    + {% for obj in queryset %} + + {% endfor %} + + + +
    +
    +{% endblock %} diff --git a/orchestra/contrib/bills/templates/admin/bills/bill/report.html b/orchestra/contrib/bills/templates/admin/bills/bill/report.html new file mode 100644 index 00000000..4a26ec78 --- /dev/null +++ b/orchestra/contrib/bills/templates/admin/bills/bill/report.html @@ -0,0 +1,87 @@ +{% load i18n utils %} + + + + Bill Report + + + + + + + + + +{% for tax, subtotal in subtotals.items %} + + + + + + + + +{% endfor %} + + + + +
    {% trans "Summary" %}{% trans "Total" %}
    {% trans "subtotal" %} {{ tax }}% {% trans "VAT" %}{{ subtotal|first}}
    {% trans "taxes" %} {{ tax }}% {% trans "VAT" %}{{ subtotal|last}}
    {% trans "TOTAL" %}{{ total }}
    + + + + + + + + + + + + +{% for bill in bills %} + + + + + + {% with base=bill.compute_base total=bill.compute_total %} + + + + {% endwith %} + +{% endfor %} +
    {% trans "Number" %}{% trans "VAT number" %}{% trans "Contact" %}{% trans "Close date" %}{% trans "Base" %}{% trans "VAT" %}{% trans "Total" %}
    {{ bill.number }}{{ bill.buyer.vat }}{{ bill.buyer.get_name }}{{ bill.closed_on|date }}{{ base }}{{ total|sub:base }}{{ total }}
    + + diff --git a/orchestra/contrib/bills/templates/admin/bills/billline/change_list.html b/orchestra/contrib/bills/templates/admin/bills/billline/change_list.html new file mode 100644 index 00000000..ea8dc4cb --- /dev/null +++ b/orchestra/contrib/bills/templates/admin/bills/billline/change_list.html @@ -0,0 +1,12 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls %} + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› {{ cl.opts.app_config.verbose_name }} +› {% trans "Bills" %} +› {% if bill %}{{ bill }}{% else %}{% trans 'Multiple bills' %}{% endif %} +› {{ cl.opts.verbose_name_plural|capfirst }} +
    +{% endblock %} diff --git a/orchestra/contrib/bills/templates/admin/bills/billline/report.html b/orchestra/contrib/bills/templates/admin/bills/billline/report.html new file mode 100644 index 00000000..c43c940c --- /dev/null +++ b/orchestra/contrib/bills/templates/admin/bills/billline/report.html @@ -0,0 +1,72 @@ +{% load i18n utils %} + + + + Transaction Report + + + + + + + + + + + + + + +{% for service, info in services %} + + + + + + + + +{% endfor %} + + + + + + + + +
    {% trans "Service" %}{% trans "Active" %}{% trans "Cancelled" %}{% trans "Nominal price" %}{% trans "Quantity" %}{% trans "Profit" %}
    {{ service }}{{ info.0 }}{{ info.1 }}{{ info.2 }}{{ info.3 }}{{ info.4 }}
    {% trans "TOTAL" %}{{ totals.0 }}{{ totals.1 }}{{ totals.2 }}{{ totals.3 }}{{ totals.4 }}
    +
    +* Custom lines +
    + + diff --git a/orchestra/contrib/bills/templates/bills/base.html b/orchestra/contrib/bills/templates/bills/base.html new file mode 100644 index 00000000..ce8a782e --- /dev/null +++ b/orchestra/contrib/bills/templates/bills/base.html @@ -0,0 +1,10 @@ + + + {% block title %}{{ bill.get_type_display }} - {{ bill.number }}{% endblock %} + + {% block head %}{% endblock %} + + + {% block body %}{% endblock %} + + diff --git a/orchestra/contrib/bills/templates/bills/bill-notification.email b/orchestra/contrib/bills/templates/bills/bill-notification.email new file mode 100644 index 00000000..7a00022a --- /dev/null +++ b/orchestra/contrib/bills/templates/bills/bill-notification.email @@ -0,0 +1,6 @@ +{% if email_part == 'subject' %}Bill {{ bill.number }}{% endif %} +{% if email_part == 'message' %}Dear {{ bill.account.username }}, +Find your {{ bill.get_type_display.lower }} attached. + +If you have any question, please write us at support@orchestra.lan +{% endif %} diff --git a/orchestra/contrib/bills/templates/bills/invoice.html b/orchestra/contrib/bills/templates/bills/invoice.html new file mode 100644 index 00000000..1b751682 --- /dev/null +++ b/orchestra/contrib/bills/templates/bills/invoice.html @@ -0,0 +1,217 @@ + + + + +

    {{ bill_type }}

    +
    + + + + + + +
    + {{ buyer.name }}
    + {{ buyer.address }}
    + {{ buyer.zipcode }} {{ buyer.city }}
    + {{ buyer.country }}
    + {{ buyer.vat_number }}
    +     		+ Invoice number
    + Date
    + Due date +     		+ : {{ bill.ident }}
    + : {{ bill.date|date:"d F, Y" }}
    + : {{ bill.due_on|date:"d F, Y" }}
    +
    +
    +
    + + + + + + + + {% for line in lines %} + + + + + + + {% endfor %} +
    ID{% trans Description %}AmountPrice
    {{ line.order_id }}{{ line.description }} + ({{ line.initial_date|date:"d-m-Y" }}{% if line.initial_date != line.final_date %} - {{ line.final_date|date:"d-m-Y" }}{% endif %}){{ line.amount }}&{{ currency }}; {{ line.price }}
    +
    +
    + + + {% for tax, base in bases.items %} + + + + {% endfor %} + + + {% for tax, value in taxes.items %} + + + + {% endfor %} + + + + + + +
     Subtotal{% if bases.items|length > 1 %} (for {{ tax }}% taxes){% endif %}&{{ currency }}; {{ base }}
     Total {{ tax }}%&{{ currency }}; {{ value }}
     Total&{{ currency }}; {{ total }}
    +
    +
    + + + + + + + + +
    IBAN + Invoice ID + Amount {{ currency.upper }} +
    NL28INGB0004954664{{ bill.ident }}{{ total }}
    +

    The invoice is to be paid before {{ invoice.exp_date|date:"F jS, Y" }} with the mention of the invoice id.

    +
    +
    + + + + + + + + +
    + {{ seller.name }}
    + {{ seller.address }}
    + {{ seller.city }}
    + {{ seller.country }}
    +     		+ Tel
    + Web
    + Email
    +     		+ {{ seller_info.phone }}
    + {{ seller_info.website }}
    + {{ seller_info.email }} +     		+ Bank ING
    + IBAN
    + BTW
    + KvK
    +     		+ 4954664
    + NL28INGB0004954664
    + NL 8207.29.449.B01
    + 27343027 +
    +
    + Payment info + + + + diff --git a/orchestra/contrib/bills/templates/bills/microspective-fee.html b/orchestra/contrib/bills/templates/bills/microspective-fee.html new file mode 100644 index 00000000..21f48170 --- /dev/null +++ b/orchestra/contrib/bills/templates/bills/microspective-fee.html @@ -0,0 +1,155 @@ +{% extends 'bills/microspective.html' %} +{% load i18n %} + +{% block head %} + +{% endblock %} + +{% block summary %} +
    +
    +
    + +
    + {{ buyer.get_name }}
    + {{ buyer.vat }}
    + {{ buyer.address }}
    + {{ buyer.zipcode }} - {{ buyer.city }}
    + {% trans buyer.get_country_display %}
    +
    + +
    + {% filter title %}{% trans bill.get_type_display %}{% endfilter %}
    + {{ bill.number }}
    + {{ bill.closed_on | default:now | date:"F j, Y" | capfirst }}
    +
    + +
    + {{ bill.compute_total }} &{{ currency.lower }};
    + {% trans "Due date" %} {{ payment.due_date| default:default_due_date | date:"F j, Y" }}
    + {% if not payment.message %}{% blocktrans with bank_account=seller_info.bank_account %}On {{ bank_account }}{% endblocktrans %}{% endif %}
    +     +
    + +
    +{% with line=bill.lines.first %} +{% blocktrans with ini=line.start_on|date:"F j, Y" end=line.end_on|date:"F j, Y" %}From {{ ini }} to {{ end }}{% endblocktrans %} +{% endwith %} +
    +{% endblock %} + + + +{% block content %} +{% block lines %} +
    +{% for line in bill.lines.all %} +
      + {% if not forloop.first %} +
    • {{ line.description }}
      • + {% endif %} +
    +{% endfor %} +
    +{% endblock %} + +{% block text %} +
    +{% blocktrans %} +With your membership you are supporting ... +{% endblocktrans %} +
    +{% endblock %} + +{% endblock %} + +{% block footer %} +
    +{{ block.super }} +{% endblock %} diff --git a/orchestra/contrib/bills/templates/bills/microspective-proforma.html b/orchestra/contrib/bills/templates/bills/microspective-proforma.html new file mode 100644 index 00000000..92400244 --- /dev/null +++ b/orchestra/contrib/bills/templates/bills/microspective-proforma.html @@ -0,0 +1,13 @@ +{% extends 'bills/microspective.html' %} + +{% block head %} + +{% endblock %} + + +{% block payment %} +{% endblock %} diff --git a/orchestra/contrib/bills/templates/bills/microspective.css b/orchestra/contrib/bills/templates/bills/microspective.css new file mode 100644 index 00000000..6513d650 --- /dev/null +++ b/orchestra/contrib/bills/templates/bills/microspective.css @@ -0,0 +1,298 @@ +body { +/* max-width: 650px;*/ + max-width: 820px; + margin: 40 auto !important; +/* margin-bottom: 30 !important;*/ + float: none !important; + font-family: sans; +} + +a { + font-size: 100%; + text-decoration: none; + vertical-align: baseline; + margin: 0; + padding: 0; + color: #666; +} + +a:hover { + color: {{ color }}; + text-decoration: underline; +} + +#logo { + float: left; + font-weight: bold; + color: {{ color }}; + margin: 1px 10px 15px 60px; + padding: 1px; +} + +#bill-number { + float: right; + text-align: right; + font-size: 20; + font-weight: bold; + color: grey; + margin-top: 30px; + margin-bottom: 10px; +} + +#bill-number .value { + font-size: 30; + color: {{ color }}; + font-weight: normal; +} + + +/* SUMMARY */ + +#bill-summary { + clear: right; +} + +#bill-summary > * { + float: right; + border: 1px solid grey; + padding: 7px 12px 7px 12px; + text-align: center; + font-size: large; + width: 100px; + overflow: hidden; + white-space: nowrap; + +} + +#bill-summary hr { + padding: 0; + margin-top: 20px; + color: #ccc; + margin-bottom: -1px; + float: none; + width: 100%; + border-left: none; + border-right: none; + border-bottom: 1px solid grey; +} + +#bill-summary .title { + color: {{ color }}; + font-size: x-small; + font-weight: bold; + position: relative; + top: -6px; +} + +#bill-summary #total, #total .title { + background-color: {{ color }}; + color: white; + font-weight: bold; +} + +#bill-summary #due-date, #bill-date, #total { + border-bottom: 2px solid grey; + height: 32px; +} + +#bill-summary #due-date { + border-right: 2px solid grey; + font-size: medium; +} + +#bill-summary #bill-date { + border-left: 2px solid grey; + font-size: medium; +} + + +/* DETAILS */ + +#seller-details, #buyer-details { + margin: 40px; +} + +#seller-details { + margin-top: 0px; +} + +#seller-details p { + margin-top: 5px; +} + +#seller-details .name { + font-weight: bold; + color: {{ color }}; +} + +#seller-details .contact { + float: left; + font-style: italic; + font-size: small; + color: #666; +} + +#buyer-details { + margin: 30px 40px 30px 60px; + font-size: 15; +} + +#buyer-details .name { + font-weight: bold; +} + + +/* LINES */ + +#lines > * { + -webkit-box-sizing: border-box; /* Safari/Chrome, other WebKit */ + -moz-box-sizing: border-box; /* Firefox, other Gecko */ + box-sizing: border-box; /* Opera/IE 8+ */ + padding-left: 10px; + float: left; + padding: 5px; + text-align: center; + font-size: small; +} + +#lines .title { + font-weight: bold; + border-bottom: 2px solid #CCC; + color: {{ color }}; +} + +#lines .last { + border-bottom: 1px solid #CCC; +} + +#lines .subline { + padding-top: 0px; +} + +#lines .column-id { + width: 8%; + text-align: right; +} + +#lines .column-description { + width: 39%; + text-align: left; +} + +#lines .column-period { + width: 23%; +} + +#lines .column-quantity { + width: 10%; +} + +#lines .column-rate { + width: 10%; +} + +#lines .column-subtotal { + width: 10%; + text-align: right; +} + + +/* TOTALS */ + +#totals { + padding-top: 100px; +} + +#totals > * { + -webkit-box-sizing: border-box; /* Safari/Chrome, other WebKit */ + -moz-box-sizing: border-box; /* Firefox, other Gecko */ + box-sizing: border-box; /* Opera/IE 8+ */ + padding: 5px; + padding-left: 10px; + text-align: right; + font-size: small; +} + +#totals .column-title { + font-weight: bold; + color: {{ color }}; + width: 86%; + float: left; +} + +#totals .column-value { + width: 14%; + float: left; +} + +#totals .subtotal { + border-bottom: 1px solid #CCC; + font-weight: normal; +} + +#totals .tax { + border-bottom: 2px solid #CCC; + font-weight: normal; +} + +#totals .total { + font-weight: bold; +} + + +/* FOOTER */ +.content { + display: table-row; /* height is dynamic, and will expand... */ + height: 100%; /* ...as content is added (won't scroll) */ +} + +.wrapper { + display: table; + height: 100%; + width: 100%; +} + +.footer { + display: table-row; +} + +.footer .title { + color: {{ color }}; + font-weight: bold; +} + +.footer > * > * { + margin: 5px; + margin-bottom: 8px; + color: #666; + font-size: small; + text-align: justify; +} + +#footer-column-1 { + float: left; + width: 48%; +} + +#footer-column-2 { + float: right; + width: 48%; +} + +#questions { + margin-bottom: 0px; +} + + +#watermark { + color: #d0d0d0; + font-size: 100pt; + -webkit-transform: rotate(-45deg); + -moz-transform: rotate(-45deg); + position: absolute; + width: 100%; + height: 100%; + margin: 0; + z-index: -1; + max-width: 593px; +} \ No newline at end of file diff --git a/orchestra/contrib/bills/templates/bills/microspective.html b/orchestra/contrib/bills/templates/bills/microspective.html new file mode 100644 index 00000000..e4422bfa --- /dev/null +++ b/orchestra/contrib/bills/templates/bills/microspective.html @@ -0,0 +1,178 @@ +{% extends 'bills/base.html' %} +{% load i18n %} + +{% block head %} + +{% endblock %} + +{% block body %} +
    +
    +{% if bill.is_open %} + +
    +

    ESBORRANY - DRAFT - BORRADOR

    +
    +{% endif %} +{% block header %} + +
    +
    + {{ seller.get_name }} +
    +
    +

    {{ seller.vat }}
    + {{ seller.address }}
    + {{ seller.zipcode }} - {% trans seller.city %}
    + {% trans seller.get_country_display %}
    +

    +

    {{ seller_info.phone }}
    + {{ seller_info.email }}
    + {{ seller_info.website }}

    +
    +
    +{% endblock %} + +{% block summary %} +
    + {% filter title %}{% trans bill.get_type_display %}{% endfilter %}
    + {{ bill.number }}
    +
    +
    +
    +
    + {% trans "DUE DATE" %}
    + {{ bill.due_on | default:default_due_date | date | capfirst }} +
    +
    + {% trans "TOTAL" %}
    + {{ bill.compute_total }} &{{ currency.lower }}; +
    +
    + {% blocktrans with bill_type=bill.get_type_display.upper %}{{ bill_type }} DATE{% endblocktrans %}
    + {{ bill.closed_on | default:now | date | capfirst }} +
    +
    +
    + {{ buyer.get_name }}
    + {{ buyer.vat }}
    + {{ buyer.address }}
    + {{ buyer.zipcode }} - {% trans buyer.city %}
    + {% trans buyer.get_country_display %}
    +
    +{% endblock %} + +{% block content %} +{% block lines %} +
    + id + {% trans "description" %} + {% trans "period" %} + {% trans "hrs/qty" %} + {% trans "rate/price" %} + {% trans "subtotal" %} +
    + {% for line in lines %} + {% with sublines=line.sublines.all description=line.description|slice:"38:" %} + {% if not line.order_id %}L{% endif %}{{ line.order_id|default:line.pk }} + {{ line.description|safe|slice:":38" }} + {{ line.get_verbose_period }} + {{ line.get_verbose_quantity|default:" "|safe }} + {% if line.rate %}{{ line.rate }} &{{ currency.lower }};{% else %} {% endif %} + {{ line.subtotal }} &{{ currency.lower }}; +
    + {% if description %} +   + {{ description|safe|truncatechars:39 }} +   +   +   +   + {% endif %} + {% for subline in sublines %} +   + {{ subline.description|safe|truncatechars:39 }} +   +   +   + {{ subline.total }} &{{ currency.lower }}; +
    + {% endfor %} + {% endwith %} + {% endfor %} +
    +{% endblock %} + +{% block totals %} +
    +
     
    + {% for tax, subtotal in bill.compute_subtotals.items %} + {% trans "subtotal" %} {{ tax }}% {% trans "VAT" %} + {{ subtotal | first }} &{{ currency.lower }}; +
    + {% trans "taxes" %} {{ tax }}% {% trans "VAT" %} + {{ subtotal | last }} &{{ currency.lower }}; +
    + {% endfor %} + {% trans "total" %} + {{ bill.compute_total }} &{{ currency.lower }}; +
    +
    +{% endblock %} +{% endblock %} + +{% block footer %} +
    +
    + + +
    +
    +{% endblock %} +{% endblock %} diff --git a/orchestra/contrib/contacts/__init__.py b/orchestra/contrib/contacts/__init__.py new file mode 100644 index 00000000..3af15748 --- /dev/null +++ b/orchestra/contrib/contacts/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.contacts.apps.ContactsConfig' diff --git a/orchestra/contrib/contacts/admin.py b/orchestra/contrib/contacts/admin.py new file mode 100644 index 00000000..c5c76471 --- /dev/null +++ b/orchestra/contrib/contacts/admin.py @@ -0,0 +1,113 @@ +from django import forms +from django.contrib import admin +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import AtLeastOneRequiredInlineFormSet, ExtendedModelAdmin +from orchestra.admin.actions import SendEmail +from orchestra.admin.utils import insertattr, change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdmin, AccountAdminMixin +from orchestra.forms.widgets import PaddingCheckboxSelectMultiple + +from .filters import EmailUsageListFilter +from .models import Contact + + +class ContactAdmin(AccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'dispaly_name', 'email', 'phone', 'phone2', 'country', 'account_link' + ) + # TODO email usage custom filter contains + list_filter = (EmailUsageListFilter,) + search_fields = ( + 'account__username', 'account__full_name', 'short_name', 'full_name', 'phone', 'phone2', + 'email' + ) + fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('account_link', 'short_name', 'full_name') + }), + (_("Email"), { + 'classes': ('wide',), + 'fields': ('email', 'email_usage',) + }), + (_("Phone"), { + 'classes': ('wide',), + 'fields': ('phone', 'phone2'), + }), + (_("Postal address"), { + 'classes': ('wide',), + 'fields': ('address', ('zipcode', 'city'), 'country') + }), + ) + # TODO don't repeat all only for account_link do it on accountadmin + add_fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('account', 'short_name', 'full_name') + }), + (_("Email"), { + 'classes': ('wide',), + 'fields': ('email', 'email_usage',) + }), + (_("Phone"), { + 'classes': ('wide',), + 'fields': ('phone', 'phone2'), + }), + (_("Postal address"), { + 'classes': ('wide',), + 'fields': ('address', ('zipcode', 'city'), 'country') + }), + ) + actions = (SendEmail(), list_accounts) + + def dispaly_name(self, contact): + return str(contact) + dispaly_name.short_description = _("Name") + dispaly_name.admin_order_field = 'short_name' + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'address': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 2}) + if db_field.name == 'email_usage': + kwargs['widget'] = PaddingCheckboxSelectMultiple(130) + return super(ContactAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + +admin.site.register(Contact, ContactAdmin) + + +class ContactInline(admin.StackedInline): + model = Contact + formset = AtLeastOneRequiredInlineFormSet + extra = 0 + fields = ( + ('short_name', 'full_name'), 'email', 'email_usage', ('phone', 'phone2'), + ) + + def get_extra(self, request, obj=None, **kwargs): + return 0 if obj and obj.contacts.exists() else 1 + + def get_view_on_site_url(self, obj=None): + if obj: + return change_url(obj) + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'short_name': + kwargs['widget'] = forms.TextInput(attrs={'size':'15'}) + if db_field.name == 'address': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 2}) + if db_field.name == 'email_usage': + kwargs['widget'] = PaddingCheckboxSelectMultiple(45) + return super(ContactInline, self).formfield_for_dbfield(db_field, **kwargs) + + +insertattr(AccountAdmin, 'inlines', ContactInline) +search_fields = ( + 'contacts__short_name', 'contacts__full_name', +) +for field in search_fields: + insertattr(AccountAdmin, 'search_fields', field) diff --git a/orchestra/contrib/contacts/api.py b/orchestra/contrib/contacts/api.py new file mode 100644 index 00000000..6a2c5ee9 --- /dev/null +++ b/orchestra/contrib/contacts/api.py @@ -0,0 +1,15 @@ +from rest_framework import viewsets + +from orchestra.api import router, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import Contact +from .serializers import ContactSerializer + + +class ContactViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = Contact.objects.all() + serializer_class = ContactSerializer + + +router.register(r'contacts', ContactViewSet) diff --git a/orchestra/contrib/contacts/apps.py b/orchestra/contrib/contacts/apps.py new file mode 100644 index 00000000..4ed7fe70 --- /dev/null +++ b/orchestra/contrib/contacts/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + +from orchestra.core import accounts + + +class ContactsConfig(AppConfig): + name = 'orchestra.contrib.contacts' + verbose_name = 'Contacts' + + def ready(self): + from .models import Contact + accounts.register(Contact, icon='contact_book.png') diff --git a/orchestra/contrib/contacts/filters.py b/orchestra/contrib/contacts/filters.py new file mode 100644 index 00000000..0f36a547 --- /dev/null +++ b/orchestra/contrib/contacts/filters.py @@ -0,0 +1,18 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + +from .models import Contact + + +class EmailUsageListFilter(SimpleListFilter): + title = _("email usages") + parameter_name = 'email_usages' + + def lookups(self, request, model_admin): + return Contact.EMAIL_USAGES + + def queryset(self, request, queryset): + value = self.value() + if value is None: + return queryset + return queryset.filter(email_usages=value.split(',')) diff --git a/orchestra/contrib/contacts/models.py b/orchestra/contrib/contacts/models.py new file mode 100644 index 00000000..2f069afb --- /dev/null +++ b/orchestra/contrib/contacts/models.py @@ -0,0 +1,80 @@ +from django.core.exceptions import ValidationError +from django.core.validators import RegexValidator +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import validators +from orchestra.models.fields import MultiSelectField + +from . import settings +from .validators import validate_phone + + +class ContactQuerySet(models.QuerySet): + def filter(self, *args, **kwargs): + usages = kwargs.pop('email_usages', []) + qs = models.Q() + for usage in usages: + qs = qs | models.Q(email_usage__regex=r'.*(^|,)+%s($|,)+.*' % usage) + return super(ContactQuerySet, self).filter(qs, *args, **kwargs) + + +class Contact(models.Model): + BILLING = 'BILLING' + EMAIL_USAGES = ( + ('SUPPORT', _("Support tickets")), + ('ADMIN', _("Administrative")), + (BILLING, _("Billing")), + ('TECH', _("Technical")), + ('ADDS', _("Announcements")), + ('EMERGENCY', _("Emergency contact")), + ) + + objects = ContactQuerySet.as_manager() + + account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), + related_name='contacts', null=True, on_delete=models.SET_NULL) + short_name = models.CharField(_("short name"), max_length=128) + full_name = models.CharField(_("full name"), max_length=256, blank=True) + email = models.EmailField() + email_usage = MultiSelectField(_("email usage"), max_length=256, blank=True, + choices=EMAIL_USAGES, + default=settings.CONTACTS_DEFAULT_EMAIL_USAGES) + phone = models.CharField(_("phone"), max_length=32, blank=True, + validators=[validate_phone]) + phone2 = models.CharField(_("alternative phone"), max_length=32, blank=True, + validators=[validate_phone]) + address = models.TextField(_("address"), blank=True) + city = models.CharField(_("city"), max_length=128, blank=True) + zipcode = models.CharField(_("zip code"), max_length=10, blank=True, + validators=[ + RegexValidator(r'^[0-9,A-Z]{3,10}$', + _("Enter a valid zipcode."), 'invalid') + ]) + country = models.CharField(_("country"), max_length=20, blank=True, + choices=settings.CONTACTS_COUNTRIES, + default=settings.CONTACTS_DEFAULT_COUNTRY) + + def __str__(self): + return self.full_name or self.short_name + + def clean(self): + self.short_name = self.short_name.strip() + self.full_name = self.full_name.strip() + self.phone = self.phone.strip() + self.phone2 = self.phone2.strip() + self.address = self.address.strip() + self.city = self.city.strip() + self.country = self.country.strip() + errors = {} + if self.address and not (self.city and self.zipcode and self.country): + errors['__all__'] = _("City, zipcode and country must be provided when address is provided.") + if self.zipcode and not self.country: + errors['country'] = _("Country must be provided when zipcode is provided.") + elif self.zipcode and self.country: + try: + validators.validate_zipcode(self.zipcode, self.country) + except ValidationError as error: + errors['zipcode'] = error + if errors: + raise ValidationError(errors) diff --git a/orchestra/contrib/contacts/serializers.py b/orchestra/contrib/contacts/serializers.py new file mode 100644 index 00000000..54538ccd --- /dev/null +++ b/orchestra/contrib/contacts/serializers.py @@ -0,0 +1,17 @@ +from rest_framework import serializers + +#from orchestra.api.serializers import MultiSelectField +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .models import Contact + + +class ContactSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + email_usage = serializers.MultipleChoiceField(choices=Contact.EMAIL_USAGES) + + class Meta: + model = Contact + fields = ( + 'url', 'id', 'short_name', 'full_name', 'email', 'email_usage', 'phone', + 'phone2', 'address', 'city', 'zipcode', 'country' + ) diff --git a/orchestra/contrib/contacts/settings.py b/orchestra/contrib/contacts/settings.py new file mode 100644 index 00000000..111231f1 --- /dev/null +++ b/orchestra/contrib/contacts/settings.py @@ -0,0 +1,32 @@ +from django_countries import data + +from orchestra.contrib.settings import Setting + + +CONTACTS_DEFAULT_EMAIL_USAGES = Setting('CONTACTS_DEFAULT_EMAIL_USAGES', + default=( + 'SUPPORT', + 'ADMIN', + 'BILLING', + 'TECH', + 'ADDS', + 'EMERGENCY' + ), +) + + +CONTACTS_DEFAULT_CITY = Setting('CONTACTS_DEFAULT_CITY', + default='Barcelona' +) + + +CONTACTS_COUNTRIES = Setting('CONTACTS_COUNTRIES', + default=tuple((k,v) for k,v in data.COUNTRIES.items()), + serializable=False +) + + +CONTACTS_DEFAULT_COUNTRY = Setting('CONTACTS_DEFAULT_COUNTRY', + default='ES', + choices=CONTACTS_COUNTRIES +) diff --git a/orchestra/contrib/contacts/validators.py b/orchestra/contrib/contacts/validators.py new file mode 100644 index 00000000..c97e4ca7 --- /dev/null +++ b/orchestra/contrib/contacts/validators.py @@ -0,0 +1,7 @@ +from orchestra.core import validators + +from . import settings + + +def validate_phone(phone): + validators.validate_phone(phone, settings.CONTACTS_DEFAULT_COUNTRY) diff --git a/orchestra/contrib/databases/__init__.py b/orchestra/contrib/databases/__init__.py new file mode 100644 index 00000000..f21f8dda --- /dev/null +++ b/orchestra/contrib/databases/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.databases.apps.DatabasesConfig' diff --git a/orchestra/contrib/databases/admin.py b/orchestra/contrib/databases/admin.py new file mode 100644 index 00000000..21ca6da9 --- /dev/null +++ b/orchestra/contrib/databases/admin.py @@ -0,0 +1,131 @@ +from django.urls import re_path as url +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin +from orchestra.admin.utils import change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import SelectAccountAdminMixin + +from .filters import HasUserListFilter, HasDatabaseListFilter +from .forms import DatabaseCreationForm, DatabaseUserChangeForm, DatabaseUserCreationForm, DatabaseForm +from .models import Database, DatabaseUser + +def save_selected(modeladmin, request, queryset): + for selected in queryset: + selected.save() +save_selected.short_description = "Re-save selected objects" + +class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): + list_display = ('name', 'type', 'display_users', 'account_link') + list_filter = ('type', HasUserListFilter) + search_fields = ('name', 'account__username') + change_readonly_fields = ('name', 'type', 'target_server') + extra = 1 + fieldsets = ( + (None, { + 'classes': ('extrapretty',), + 'fields': ('account_link', 'name', 'type', 'users', 'display_users', 'comments', 'target_server'), + }), + ) + add_fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('account_link', 'name', 'type', 'target_server') + }), + (_("Create new user"), { + 'classes': ('wide',), + 'fields': ('username', 'password1', 'password2'), + }), + (_("Use existing user"), { + 'classes': ('wide',), + 'fields': ('user',) + }), + ) + form = DatabaseForm + add_form = DatabaseCreationForm + readonly_fields = ('account_link', 'display_users',) + filter_horizontal = ['users'] + # filter_by_account_fields = ('users',) + # list_prefetch_related = ('users',) + actions = (list_accounts, save_selected) + + @mark_safe + def display_users(self, db): + links = [] + for user in db.users.all(): + link = format_html('{}', change_url(user), user.username) + links.append(link) + return '
    '.join(links) + display_users.short_description = _("Users") + display_users.admin_order_field = 'users__username' + + def save_model(self, request, obj, form, change): + super(DatabaseAdmin, self).save_model(request, obj, form, change) + if not change: + user = form.cleaned_data['user'] + if not user: + user = DatabaseUser( + username=form.cleaned_data['username'], + type=obj.type, + account_id=obj.account.pk, + target_server=form.cleaned_data['target_server'], + ) + user.set_password(form.cleaned_data["password1"]) + user.save() + obj.users.add(user) + + +class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, ExtendedModelAdmin): + list_display = ('username', 'target_server', 'type', 'display_databases', 'account_link') + list_filter = ('type', HasDatabaseListFilter) + search_fields = ('username', 'account__username') + form = DatabaseUserChangeForm + add_form = DatabaseUserCreationForm + change_readonly_fields = ('username', 'type', 'target_server') + fieldsets = ( + (None, { + 'classes': ('extrapretty',), + 'fields': ('account_link', 'username', 'password', 'type', 'display_databases', 'target_server', 'permision') + }), + ) + add_fieldsets = ( + (None, { + 'classes': ('extrapretty',), + 'fields': ('account_link', 'username', 'password1', 'password2', 'type', 'target_server', 'permision') + }), + ) + readonly_fields = ('account_link', 'display_databases',) + filter_by_account_fields = ('databases',) + list_prefetch_related = ('databases',) + actions = (list_accounts, save_selected) + + @mark_safe + def display_databases(self, user): + links = [] + for db in user.databases.all(): + link = format_html('{}', change_url(db), db.name) + links.append(link) + return '
    '.join(links) + display_databases.short_description = _("Databases") + display_databases.admin_order_field = 'databases__name' + + def get_urls(self): + useradmin = UserAdmin(DatabaseUser, self.admin_site) + return [ + url(r'^(\d+)/password/$', + self.admin_site.admin_view(useradmin.user_change_password)) + ] + super(DatabaseUserAdmin, self).get_urls() + + def save_model(self, request, obj, form, change): + """ set password """ + if not change: + obj.set_password(form.cleaned_data["password1"]) + super(DatabaseUserAdmin, self).save_model(request, obj, form, change) + + +admin.site.register(Database, DatabaseAdmin) +admin.site.register(DatabaseUser, DatabaseUserAdmin) diff --git a/orchestra/contrib/databases/api.py b/orchestra/contrib/databases/api.py new file mode 100644 index 00000000..808d02c0 --- /dev/null +++ b/orchestra/contrib/databases/api.py @@ -0,0 +1,23 @@ +from rest_framework import viewsets + +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import Database, DatabaseUser +from .serializers import DatabaseSerializer, DatabaseUserSerializer + + +class DatabaseViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = Database.objects.prefetch_related('users').all() + serializer_class = DatabaseSerializer + filter_fields = ('name',) + + +class DatabaseUserViewSet(LogApiMixin, AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): + queryset = DatabaseUser.objects.prefetch_related('databases').all() + serializer_class = DatabaseUserSerializer + filter_fields = ('username',) + + +router.register(r'databases', DatabaseViewSet) +router.register(r'databaseusers', DatabaseUserViewSet) diff --git a/orchestra/contrib/databases/apps.py b/orchestra/contrib/databases/apps.py new file mode 100644 index 00000000..87e89386 --- /dev/null +++ b/orchestra/contrib/databases/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import services + + +class DatabasesConfig(AppConfig): + name = 'orchestra.contrib.databases' + verbose_name = 'Databases' + + def ready(self): + from .models import Database, DatabaseUser + services.register(Database, icon='database.png') + services.register(DatabaseUser, icon='postgresql.png', verbose_name_plural=_("Database users")) diff --git a/orchestra/contrib/databases/backends.py b/orchestra/contrib/databases/backends.py new file mode 100644 index 00000000..ca48ff03 --- /dev/null +++ b/orchestra/contrib/databases/backends.py @@ -0,0 +1,193 @@ +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace +from orchestra.contrib.resources import ServiceMonitor + +from . import settings + + +class MySQLController(ServiceController): + """ + Simple backend for creating MySQL databases using CREATE DATABASE statement. + """ + verbose_name = "MySQL database" + model = 'databases.Database' + default_route_match = "database.type == 'mysql'" + doc_settings = (settings, + ('DATABASES_DEFAULT_HOST',) + ) + + def save(self, database): + if database.type != database.MYSQL: + return + context = self.get_context(database) + # Not available on delete() + context['owner'] = database.owner + self.append(textwrap.dedent(""" + # Create database and re-set permissions + mysql -e 'CREATE DATABASE `%(database)s`;' || true + mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'\ + """) % context + ) + for user in database.users.all(): + context.update({ + 'username': user.username, + 'grant': 'WITH GRANT OPTION' if user == context['owner'] else '' + }) + if user.permision == "ro": + self.append(textwrap.dedent("""\ + mysql -e 'GRANT SELECT ON `%(database)s`.* TO "%(username)s"@"%(host)s" %(grant)s;'\ + """) % context + ) + else: + self.append(textwrap.dedent("""\ + mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(username)s"@"%(host)s" %(grant)s;'\ + """) % context + ) + + def delete(self, database): + if database.type != database.MYSQL: + return + context = self.get_context(database) + self.append(textwrap.dedent(""" + # Remove database %(database)s + mysql -e 'DROP DATABASE `%(database)s`;' || exit_code=$? + mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'\ + """) % context + ) + + def commit(self): + self.append(textwrap.dedent(""" + # Apply permissions + mysql -e 'FLUSH PRIVILEGES;'\ + """) + ) + super(MySQLController, self).commit() + + def get_context(self, database): + context = { + 'database': database.name, + 'host': settings.DATABASES_DEFAULT_HOST, + } + return replace(replace(context, "'", '"'), ';', '') + + +class MySQLUserController(ServiceController): + """ + Simple backend for creating MySQL users using CREATE USER statement. + """ + verbose_name = "MySQL user" + model = 'databases.DatabaseUser' + default_route_match = "databaseuser.type == 'mysql'" + doc_settings = (settings, + ('DATABASES_DEFAULT_HOST',) + ) + + def save(self, user): + if user.type != user.MYSQL: + return + context = self.get_context(user) + if user.target_server.name != "mysql.pangea.lan": + self.append(textwrap.dedent("""\ + # Create user %(username)s + mysql -e 'CREATE USER IF NOT EXISTS "%(username)s"@"%(host)s";' + mysql -e 'ALTER USER IF EXISTS "%(username)s"@"%(host)s" IDENTIFIED BY PASSWORD "%(password)s";'\ + """) % context + ) + else: + self.append(textwrap.dedent("""\ + # Create user %(username)s + mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true # User already exists + mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";'\ + """) % context + ) + + def delete(self, user): + if user.type != user.MYSQL: + return + context = self.get_context(user) + self.append(textwrap.dedent(""" + # Delete user %(username)s + mysql -e 'DROP USER "%(username)s"@"%(host)s";' || exit_code=$? \ + """) % context + ) + + def commit(self): + self.append("# Apply permissions") + self.append("mysql -e 'FLUSH PRIVILEGES;'") + + def get_context(self, user): + context = { + 'username': user.username, + 'password': user.password, + 'host': settings.DATABASES_DEFAULT_HOST, + } + return replace(replace(context, "'", '"'), ';', '') + + +class MysqlDisk(ServiceMonitor): + """ + du -bs <database_path> + Implements triggers for resource limit exceeded and recovery, disabling insert and create privileges. + """ + model = 'databases.Database' + verbose_name = _("MySQL disk") + delete_old_equal_values = True + doc_settings = (settings, + ('DATABASES_MYSQL_DB_DIR',) + ) + mysql_db_dir = settings.DATABASES_MYSQL_DB_DIR + + def exceeded(self, db): + if db.type != db.MYSQL: + return + context = self.get_context(db) + self.append(textwrap.dedent("""\ + mysql -e 'UPDATE db SET Insert_priv="N", Create_priv="N" WHERE Db="%(db_name)s";'\ + """) % context + ) + + def recovery(self, db): + if db.type != db.MYSQL: + return + context = self.get_context(db) + self.append(textwrap.dedent("""\ + mysql -e 'UPDATE db SET Insert_priv="Y", Create_priv="Y" WHERE Db="%(db_name)s";'\ + """) % context + ) + + def prepare(self): + super().prepare() + context = { + 'mysql_db_dir': self.mysql_db_dir, + } + self.append(textwrap.dedent("""\ + function monitor_mysql () { + { SIZE=$(du -bs "%(mysql_db_dir)s/$1") && echo $SIZE || echo 0; } | awk {'print $1'} + }""") % context) + # Slower way + #self.append(textwrap.dedent("""\ + # function monitor () { + # mysql -B -e " + # SELECT IFNULL(sum(data_length + index_length), 0) 'Size' + # FROM information_schema.TABLES + # WHERE table_schema = '$1'; + # " | tail -n 1 + # }""")) + + def monitor(self, db): + if db.type != db.MYSQL: + return + context = self.get_context(db) + self.append('echo %(db_id)s $(monitor_%(db_type)s "%(db_dirname)s")' % context) + + def get_context(self, db): + context = { + 'db_name': db.name, + 'db_dirname': db.name.replace('-', '@002d'), + 'db_id': db.pk, + 'db_type': db.type, + } + return replace(replace(context, "'", '"'), ';', '') diff --git a/orchestra/contrib/databases/filters.py b/orchestra/contrib/databases/filters.py new file mode 100644 index 00000000..50136c70 --- /dev/null +++ b/orchestra/contrib/databases/filters.py @@ -0,0 +1,34 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + + +class HasUserListFilter(SimpleListFilter): + """ Filter addresses whether they have any db user or not """ + title = _("has user") + parameter_name = 'has_user' + + def lookups(self, request, model_admin): + return ( + ('True', _("True")), + ('False', _("False")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(users__isnull=False) + elif self.value() == 'False': + return queryset.filter(users__isnull=True) + return queryset + + +class HasDatabaseListFilter(HasUserListFilter): + """ Filter addresses whether they have any db or not """ + title = _("has database") + parameter_name = 'has_database' + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(databases__isnull=False) + elif self.value() == 'False': + return queryset.filter(databases__isnull=True) + return queryset diff --git a/orchestra/contrib/databases/forms.py b/orchestra/contrib/databases/forms.py new file mode 100644 index 00000000..f35f00c7 --- /dev/null +++ b/orchestra/contrib/databases/forms.py @@ -0,0 +1,153 @@ +from django import forms +from django.contrib.auth.forms import ReadOnlyPasswordHashField +from django.core.exceptions import ValidationError +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import validators + +from .models import DatabaseUser, Database + + +class DatabaseUserCreationForm(forms.ModelForm): + password1 = forms.CharField(label=_("Password"), required=False, + widget=forms.PasswordInput(attrs={'autocomplete': 'off'}), + validators=[validators.validate_password]) + password2 = forms.CharField(label=_("Password confirmation"), required=False, + widget=forms.PasswordInput, + help_text=_("Enter the same password as above, for verification.")) + + class Meta: + model = DatabaseUser + fields = ('username', 'account', 'type') + + def clean_password2(self): + password1 = self.cleaned_data.get("password1") + password2 = self.cleaned_data.get("password2") + if password1 and password2 and password1 != password2: + msg = _("The two password fields didn't match.") + raise ValidationError(msg) + return password2 + + +class DatabaseForm(forms.ModelForm): + + class Meta: + model = Database + fields = ('name', 'users', 'type', 'account', 'target_server') + + def __init__(self, *args, **kwargs): + super(DatabaseForm, self).__init__(*args, **kwargs) + # muestra solo los usuarios del mismo server + account_id = self.instance.account_id + database_server_id = self.instance.target_server_id + if account_id: + self.fields['users'].queryset = DatabaseUser.objects.filter(account=account_id, target_server=database_server_id) + + def clean(self): + # verifica que los usuarios petenecen al servidor de la bbdd + database_server_id = self.instance.target_server_id + users = self.cleaned_data.get('users') + if users and database_server_id: + for user in users: + if user.target_server_id != database_server_id: + self.add_error("users", _(f"{user.username} does not belong to the database server")) + + return self.cleaned_data + + +class DatabaseCreationForm(DatabaseUserCreationForm): + username = forms.CharField(label=_("Username"), max_length=16, + required=False, validators=[validators.validate_name], + help_text=_("Required. 16 characters or fewer. Letters, digits and " + "@/./+/-/_ only."), + error_messages={ + 'invalid': _("This value may contain 16 characters or fewer, only letters, numbers and " + "@/./+/-/_ characters.")}) + user = forms.ModelChoiceField(required=False, queryset=DatabaseUser.objects) + + class Meta: + model = Database + fields = ('username', 'account', 'type') + + def __init__(self, *args, **kwargs): + super(DatabaseCreationForm, self).__init__(*args, **kwargs) + account_id = self.initial.get('account', self.initial_account) + if account_id: + qs = self.fields['user'].queryset.filter(account=account_id).order_by('username') + choices = [ (u.pk, "%s (%s) (%s)" % (u, u.get_type_display(), str(u.target_server.name) )) for u in qs ] + self.fields['user'].queryset = qs + self.fields['user'].choices = [(None, '--------'),] + choices + + def clean_username(self): + username = self.cleaned_data.get('username') + server = self.cleaned_data.get('target_server') + if DatabaseUser.objects.filter(username=username, target_server=server).exists(): + raise ValidationError("Provided username already exists.") + return username + + def clean_password2(self): + username = self.cleaned_data.get('username') + password1 = self.cleaned_data.get('password1') + password2 = self.cleaned_data.get('password2') + if username and not (password1 and password2): + raise ValidationError(_("Missing password")) + if password1 and password2 and password1 != password2: + msg = _("The two password fields didn't match.") + raise ValidationError(msg) + return password2 + + def clean_user(self): + user = self.cleaned_data.get('user') + if user and user.type != self.cleaned_data.get('type'): + msg = _("Database type and user type doesn't match") + raise ValidationError(msg) + if user and user.target_server != self.cleaned_data.get('target_server'): + msg = _("Database server and user server doesn't match") + raise ValidationError(msg) + return user + + def clean(self): + cleaned_data = super(DatabaseCreationForm, self).clean() + if 'user' in cleaned_data and 'username' in cleaned_data: + msg = _("Use existing user or create a new one? you have provided both.") + if cleaned_data['user'] and self.cleaned_data['username']: + raise ValidationError(msg) + elif not (cleaned_data['username'] or cleaned_data['user']): + raise ValidationError(msg) + return cleaned_data + + +class ReadOnlySQLPasswordHashField(ReadOnlyPasswordHashField): + class ReadOnlyPasswordHashWidget(forms.Widget): + def render(self, name, value, attrs, renderer=None): + original = ReadOnlyPasswordHashField.widget().render(name, value, attrs) + if 'Invalid' not in original: + return original + encoded = value + if not encoded: + summary = mark_safe("%s" % _("No password set.")) + else: + size = len(value) + summary = value[:int(size/2)] + '*'*int(size-size/2) + summary = "hash: %s" % summary + if value.startswith('*'): + summary = "algorithm: sha1_bin_hex %s" % summary + return format_html("
    %s
    " % summary) + widget = ReadOnlyPasswordHashWidget + + +class DatabaseUserChangeForm(forms.ModelForm): + password = ReadOnlySQLPasswordHashField(label=_("Password"), + help_text=_("Raw passwords are not stored, so there is no way to see " + "this user's password, but you can change the password " + "using this form. " + "Show hash.")) + + class Meta: + model = DatabaseUser + fields = ('username', 'password', 'type', 'account') + + def clean_password(self): + return self.initial["password"] diff --git a/orchestra/contrib/databases/migrations/0001_initial.py b/orchestra/contrib/databases/migrations/0001_initial.py new file mode 100644 index 00000000..cf3f8f85 --- /dev/null +++ b/orchestra/contrib/databases/migrations/0001_initial.py @@ -0,0 +1,46 @@ +# Generated by Django 2.2.28 on 2023-06-28 17:06 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import orchestra.core.validators + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='DatabaseUser', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('username', models.CharField(max_length=16, validators=[orchestra.core.validators.validate_name], verbose_name='username')), + ('password', models.CharField(max_length=256, verbose_name='password')), + ('type', models.CharField(choices=[('mysql', 'MySQL')], default='mysql', max_length=32, verbose_name='type')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='databaseusers', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ], + options={ + 'verbose_name_plural': 'DB users', + 'unique_together': {('username', 'type')}, + }, + ), + migrations.CreateModel( + name='Database', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=64, validators=[orchestra.core.validators.validate_name], verbose_name='name')), + ('type', models.CharField(choices=[('mysql', 'MySQL')], default='mysql', max_length=32, verbose_name='type')), + ('comments', models.TextField(blank=True, default='')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='databases', to=settings.AUTH_USER_MODEL, verbose_name='Account')), + ('users', models.ManyToManyField(blank=True, related_name='databases', to='databases.DatabaseUser', verbose_name='users')), + ], + options={ + 'unique_together': {('name', 'type')}, + }, + ), + ] diff --git a/orchestra/contrib/databases/migrations/0002_databaseuser_target_server.py b/orchestra/contrib/databases/migrations/0002_databaseuser_target_server.py new file mode 100644 index 00000000..3db3d0ca --- /dev/null +++ b/orchestra/contrib/databases/migrations/0002_databaseuser_target_server.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.28 on 2023-06-28 17:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '__first__'), + ('databases', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='databaseuser', + name='target_server', + field=models.ForeignKey(default=3, on_delete=django.db.models.deletion.CASCADE, to='orchestration.Server', verbose_name='Target Server'), + ), + ] diff --git a/orchestra/contrib/databases/migrations/0003_auto_20230629_1838.py b/orchestra/contrib/databases/migrations/0003_auto_20230629_1838.py new file mode 100644 index 00000000..cd71691f --- /dev/null +++ b/orchestra/contrib/databases/migrations/0003_auto_20230629_1838.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.28 on 2023-06-29 16:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '__first__'), + ('databases', '0002_databaseuser_target_server'), + ] + + operations = [ + migrations.AddField( + model_name='databaseuser', + name='permision', + field=models.CharField(choices=[('all', 'all'), ('ro', 'read only')], default='all', max_length=20, verbose_name='Permisson'), + ), + migrations.AlterField( + model_name='databaseuser', + name='target_server', + field=models.ForeignKey(default=3, on_delete=django.db.models.deletion.CASCADE, to='orchestration.Server', verbose_name='Server'), + ), + migrations.AlterUniqueTogether( + name='databaseuser', + unique_together={('username', 'type', 'target_server')}, + ), + ] diff --git a/orchestra/contrib/databases/migrations/0004_database_target_server.py b/orchestra/contrib/databases/migrations/0004_database_target_server.py new file mode 100644 index 00000000..2868cbfb --- /dev/null +++ b/orchestra/contrib/databases/migrations/0004_database_target_server.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.28 on 2023-06-29 16:57 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '__first__'), + ('databases', '0003_auto_20230629_1838'), + ] + + operations = [ + migrations.AddField( + model_name='database', + name='target_server', + field=models.ForeignKey(default=3, on_delete=django.db.models.deletion.CASCADE, to='orchestration.Server', verbose_name='Server'), + ), + ] diff --git a/orchestra/contrib/databases/migrations/0005_auto_20230705_1208.py b/orchestra/contrib/databases/migrations/0005_auto_20230705_1208.py new file mode 100644 index 00000000..55285684 --- /dev/null +++ b/orchestra/contrib/databases/migrations/0005_auto_20230705_1208.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2023-07-05 10:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '__first__'), + ('databases', '0004_database_target_server'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='database', + unique_together={('name', 'type', 'target_server')}, + ), + ] diff --git a/orchestra/contrib/databases/migrations/0006_auto_20230705_1237.py b/orchestra/contrib/databases/migrations/0006_auto_20230705_1237.py new file mode 100644 index 00000000..a0864891 --- /dev/null +++ b/orchestra/contrib/databases/migrations/0006_auto_20230705_1237.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.28 on 2023-07-05 10:37 + +from django.db import migrations, models +import orchestra.core.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('databases', '0005_auto_20230705_1208'), + ] + + operations = [ + migrations.AlterField( + model_name='databaseuser', + name='username', + field=models.CharField(max_length=32, validators=[orchestra.core.validators.validate_name], verbose_name='username'), + ), + ] diff --git a/orchestra/contrib/databases/migrations/__init__.py b/orchestra/contrib/databases/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/databases/models.py b/orchestra/contrib/databases/models.py new file mode 100644 index 00000000..528657f6 --- /dev/null +++ b/orchestra/contrib/databases/models.py @@ -0,0 +1,94 @@ +import hashlib + +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import validators + +from . import settings + + +class Database(models.Model): + """ Represents a basic database for a web application """ + MYSQL = 'mysql' + POSTGRESQL = 'postgresql' + + name = models.CharField(_("name"), max_length=64, # MySQL limit + validators=[validators.validate_name]) + users = models.ManyToManyField('databases.DatabaseUser', blank=True, + verbose_name=_("users"),related_name='databases') + type = models.CharField(_("type"), max_length=32, + choices=settings.DATABASES_TYPE_CHOICES, + default=settings.DATABASES_DEFAULT_TYPE) + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='databases') + comments = models.TextField(default="", blank=True) + target_server = models.ForeignKey('orchestration.Server', on_delete=models.CASCADE, + verbose_name=_("Server"), default=3 ) + + class Meta: + unique_together = ('name', 'type', 'target_server') + + def __str__(self): + return "%s" % self.name + + @property + def owner(self): + """ database owner is the first user related to it """ + # Accessing intermediary model to get which is the first user + users = Database.users.through.objects.filter(database_id=self.id) + user = users.order_by('id').first() + if user is not None: + return user.databaseuser + return None + + @property + def active(self): + return self.account.is_active + + +Database.users.through._meta.unique_together = ( + ('database', 'databaseuser'), +) + + +class DatabaseUser(models.Model): + MYSQL = Database.MYSQL + POSTGRESQL = Database.POSTGRESQL + + typeOfPermision = [ + ('all','all'), + ('ro', 'read only'), + ] + + username = models.CharField(_("username"), max_length=32, # MySQL usernames 16 char long + validators=[validators.validate_name]) + password = models.CharField(_("password"), max_length=256) + type = models.CharField(_("type"), max_length=32, + choices=settings.DATABASES_TYPE_CHOICES, + default=settings.DATABASES_DEFAULT_TYPE) + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='databaseusers') + target_server = models.ForeignKey('orchestration.Server', on_delete=models.CASCADE, + verbose_name=_("Server"), default=3 ) + permision = models.CharField(verbose_name=_("Permisson"), max_length=20, choices=typeOfPermision, default='all') + + + class Meta: + verbose_name_plural = _("DB users") + unique_together = ('username', 'type', 'target_server') + + def __str__(self): + return self.username + + def get_username(self): + return self.username + + def set_password(self, password): + if self.type == self.MYSQL: + # MySQL stores sha1(sha1(password).binary).hex + binary = hashlib.sha1(password.encode('utf-8')).digest() + hexdigest = hashlib.sha1(binary).hexdigest() + self.password = '*%s' % hexdigest.upper() + else: + raise TypeError("Database type '%s' not supported" % self.type) diff --git a/orchestra/contrib/databases/serializers.py b/orchestra/contrib/databases/serializers.py new file mode 100644 index 00000000..7f556193 --- /dev/null +++ b/orchestra/contrib/databases/serializers.py @@ -0,0 +1,51 @@ +from rest_framework import serializers + +from orchestra.api.serializers import (HyperlinkedModelSerializer, + SetPasswordHyperlinkedSerializer, RelatedHyperlinkedModelSerializer) +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .models import Database, DatabaseUser + + +class RelatedDatabaseUserSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = DatabaseUser + fields = ('url', 'id', 'username') + + +class DatabaseSerializer(AccountSerializerMixin, HyperlinkedModelSerializer): + users = RelatedDatabaseUserSerializer(many=True) #allow_add_remove=True + + class Meta: + model = Database + fields = ('url', 'id', 'name', 'type', 'users') + postonly_fields = ('name', 'type') + + def validate(self, attrs): + attrs = super(DatabaseSerializer, self).validate(attrs) + for user in attrs['users']: + if user.type != attrs['type']: + raise serializers.ValidationError("User type must be" % attrs['type']) + return attrs + + +class RelatedDatabaseSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = Database + fields = ('url', 'id', 'name',) + + +class DatabaseUserSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): + databases = RelatedDatabaseSerializer(many=True, required=False) # allow_add_remove=True + + class Meta: + model = DatabaseUser + fields = ('url', 'id', 'username', 'password', 'type', 'databases') + postonly_fields = ('username', 'type', 'password') + + def validate(self, attrs): + attrs = super(DatabaseUserSerializer, self).validate(attrs) + for database in attrs.get('databases', []): + if database.type != attrs['type']: + raise serializers.ValidationError("Database type must be" % attrs['type']) + return attrs diff --git a/orchestra/contrib/databases/settings.py b/orchestra/contrib/databases/settings.py new file mode 100644 index 00000000..473c48a8 --- /dev/null +++ b/orchestra/contrib/databases/settings.py @@ -0,0 +1,29 @@ +from orchestra.core.validators import validate_hostname + +from orchestra.contrib.settings import Setting + + +DATABASES_TYPE_CHOICES = Setting('DATABASES_TYPE_CHOICES', + ( + ('mysql', 'MySQL'), + ('postgres', 'PostgreSQL'), + ), + validators=[Setting.validate_choices] +) + + +DATABASES_DEFAULT_TYPE = Setting('DATABASES_DEFAULT_TYPE', + 'mysql', + choices=DATABASES_TYPE_CHOICES, +) + + +DATABASES_DEFAULT_HOST = Setting('DATABASES_DEFAULT_HOST', + 'localhost', +# validators=[validate_hostname], +) + + +DATABASES_MYSQL_DB_DIR = Setting('DATABASES_MYSQL_DB_DIR', + '/var/lib/mysql', +) diff --git a/orchestra/contrib/databases/tests/__init__.py b/orchestra/contrib/databases/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/databases/tests/functional_tests/__init__.py b/orchestra/contrib/databases/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/databases/tests/functional_tests/tests.py b/orchestra/contrib/databases/tests/functional_tests/tests.py new file mode 100644 index 00000000..3d89bff1 --- /dev/null +++ b/orchestra/contrib/databases/tests/functional_tests/tests.py @@ -0,0 +1,348 @@ +import os +import socket +import time +import unittest + +import MySQLdb +from django.conf import settings as djsettings +from django.core.management.base import CommandError +from django.urls import reverse +from orchestra.admin.utils import change_url +from orchestra.contrib.orchestration.models import Route, Server +from orchestra.utils.sys import sshrun +from orchestra.utils.tests import (BaseLiveServerTestCase, random_ascii, + save_response_on_error, snapshot_on_error) +from selenium.webdriver.support.select import Select + +from ... import backends, settings +from ...models import Database, DatabaseUser + +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + + +class DatabaseTestMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_SECOND_SERVER', 'localhost') + DEPENDENCIES = ( + 'orchestra.contrib.orchestration', + 'orcgestra.apps.databases', + ) + + def setUp(self): + super(DatabaseTestMixin, self).setUp() + self.add_route() + djsettings.DEBUG = True + + def add_route(self): + raise NotImplementedError + + def save(self): + raise NotImplementedError + + def add(self): + raise NotImplementedError + + def delete(self): + raise NotImplementedError + + def update(self): + raise NotImplementedError + + def disable(self): + raise NotImplementedError + + def add_group(self, username, groupname): + raise NotImplementedError + + def test_add(self): + dbname = '%s_database' % random_ascii(5) + username = '%s_dbuser' % random_ascii(5) + password = '@!?%spppP001' % random_ascii(5) + self.add(dbname, username, password) + self.validate_create_table(dbname, username, password) + + def test_delete(self): + dbname = '%s_database' % random_ascii(5) + username = '%s_dbuser' % random_ascii(5) + password = '@!?%spppP001' % random_ascii(5) + self.add(dbname, username, password) + self.validate_create_table(dbname, username, password) + self.delete(dbname) + self.delete_user(username) + self.validate_delete(dbname, username, password) + self.validate_delete_user(dbname, username) + + def test_change_password(self): + dbname = '%s_database' % random_ascii(5) + username = '%s_dbuser' % random_ascii(5) + password = '@!?%spppP001' % random_ascii(5) + self.add(dbname, username, password) + self.addCleanup(self.delete, dbname) + self.addCleanup(self.delete_user, username) + self.validate_create_table(dbname, username, password) + new_password = '@!?%spppP001' % random_ascii(5) + self.change_password(username, new_password) + self.validate_login_error(dbname, username, password) + self.validate_create_table(dbname, username, new_password) + + def test_add_user(self): + dbname = '%s_database' % random_ascii(5) + username = '%s_dbuser' % random_ascii(5) + password = '@!?%spppP001' % random_ascii(5) + self.add(dbname, username, password) + self.addCleanup(self.delete, dbname) + self.addCleanup(self.delete_user, username) + self.validate_create_table(dbname, username, password) + username2 = '%s_dbuser' % random_ascii(5) + password2 = '@!?%spppP001' % random_ascii(5) + self.add_user(username2, password2) + self.addCleanup(self.delete_user, username2) + self.validate_login_error(dbname, username2, password2) + self.add_user_to_db(username2, dbname) + self.validate_create_table(dbname, username, password) + self.validate_create_table(dbname, username2, password2) + + def test_delete_user(self): + dbname = '%s_database' % random_ascii(5) + username = '%s_dbuser' % random_ascii(5) + password = '@!?%spppP001' % random_ascii(5) + self.add(dbname, username, password) + self.addCleanup(self.delete, dbname) + self.validate_create_table(dbname, username, password) + username2 = '%s_dbuser' % random_ascii(5) + password2 = '@!?%spppP001' % random_ascii(5) + self.add_user(username2, password2) + self.add_user_to_db(username2, dbname) + self.delete_user(username) + self.validate_delete_user(username, password) + self.validate_login_error(dbname, username, password) + self.validate_create_table(dbname, username2, password2) + self.delete_user(username2) + self.validate_login_error(dbname, username2, password2) + self.validate_delete_user(username2, password2) + + def test_swap_user(self): + dbname = '%s_database' % random_ascii(5) + username = '%s_dbuser' % random_ascii(5) + password = '@!?%spppP001' % random_ascii(5) + self.add(dbname, username, password) + self.addCleanup(self.delete, dbname) + self.addCleanup(self.delete_user, username) + self.validate_create_table(dbname, username, password) + username2 = '%s_dbuser' % random_ascii(5) + password2 = '@!?%spppP001' % random_ascii(5) + self.add_user(username2, password2) + self.addCleanup(self.delete_user, username2) + self.swap_user(username, username2, dbname) + self.validate_login_error(dbname, username, password) + self.validate_create_table(dbname, username2, password2) + + +class MySQLControllerMixin(object): + db_type = 'mysql' + + def setUp(self): + super(MySQLControllerMixin, self).setUp() + # Get local ip address used to reach self.MASTER_SERVER + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect((self.MASTER_SERVER, 22)) + settings.DATABASES_DEFAULT_HOST = s.getsockname()[0] + s.close() + + def add_route(self): + server = Server.objects.create(name=self.MASTER_SERVER) + backend = backends.MySQLController.get_name() + match = "database.type == '%s'" % self.db_type + Route.objects.create(backend=backend, match=match, host=server) + match = "databaseuser.type == '%s'" % self.db_type + backend = backends.MySQLUserController.get_name() + Route.objects.create(backend=backend, match=match, host=server) + + def validate_create_table(self, name, username, password): + db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name) + cur = db.cursor() + cur.execute('CREATE TABLE table_%s ( id INT ) ;' % random_ascii(10)) + + def validate_login_error(self, dbname, username, password): + self.assertRaises(MySQLdb.OperationalError, + self.validate_create_table, dbname, username, password + ) + + def validate_delete(self, dbname, username, password): + self.validate_login_error(dbname, username, password) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'mysql %s' % dbname, display=False) + + def validate_delete_user(self, name, username): + context = { + 'name': name, + 'username': username, + } + self.assertEqual('', sshrun(self.MASTER_SERVER, + """mysql mysql -e 'SELECT * FROM db WHERE db="%(name)s";'""" % context, display=False).stdout) + self.assertEqual('', sshrun(self.MASTER_SERVER, + """mysql mysql -e 'SELECT * FROM user WHERE user="%(username)s";'""" % context, display=False).stdout) + + +@unittest.skipUnless(TEST_REST_API, "REST API tests") +class RESTDatabaseMixin(DatabaseTestMixin): + def setUp(self): + super(RESTDatabaseMixin, self).setUp() + self.rest_login() + + @save_response_on_error + def add(self, dbname, username, password): + user = self.rest.databaseusers.create(username=username, password=password, type=self.db_type) + users = [{ + 'username': user.username + }] + self.rest.databases.create(name=dbname, users=users, type=self.db_type) + + @save_response_on_error + def delete(self, dbname): + self.rest.databases.retrieve(name=dbname).delete() + + @save_response_on_error + def change_password(self, username, password): + user = self.rest.databaseusers.retrieve(username=username).get() + user.set_password(password) + + @save_response_on_error + def add_user(self, username, password): + self.rest.databaseusers.create(username=username, password=password, type=self.db_type) + + @save_response_on_error + def add_user_to_db(self, username, dbname): + user = self.rest.databaseusers.retrieve(username=username).get() + db = self.rest.databases.retrieve(name=dbname).get() + db.users.append(user) + db.save() + + @save_response_on_error + def delete_user(self, username): + self.rest.databaseusers.retrieve(username=username).delete() + + @save_response_on_error + def swap_user(self, username, username2, dbname): + user = self.rest.databaseusers.retrieve(username=username2).get() + db = self.rest.databases.retrieve(name=dbname).get() + db.users = db.users.exclude(username=username) + db.users.append(user) + db.save() + + +class AdminDatabaseMixin(DatabaseTestMixin): + def setUp(self): + super(AdminDatabaseMixin, self).setUp() + self.admin_login() + + @snapshot_on_error + def add(self, dbname, username, password): + url = self.live_server_url + reverse('admin:databases_database_add') + self.selenium.get(url) + + type_input = self.selenium.find_element_by_id('id_type') + type_select = Select(type_input) + type_select.select_by_value(self.db_type) + + name_field = self.selenium.find_element_by_id('id_name') + name_field.send_keys(dbname) + + username_field = self.selenium.find_element_by_id('id_username') + username_field.send_keys(username) + + password_field = self.selenium.find_element_by_id('id_password1') + password_field.send_keys(password) + password_field = self.selenium.find_element_by_id('id_password2') + password_field.send_keys(password) + + name_field.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def delete(self, dbname): + db = Database.objects.get(name=dbname) + self.admin_delete(db) + + @snapshot_on_error + def change_password(self, username, password): + user = DatabaseUser.objects.get(username=username) + self.admin_change_password(user, password) + + @snapshot_on_error + def add_user(self, username, password): + url = self.live_server_url + reverse('admin:databases_databaseuser_add') + self.selenium.get(url) + + type_input = self.selenium.find_element_by_id('id_type') + type_select = Select(type_input) + type_select.select_by_value(self.db_type) + + username_field = self.selenium.find_element_by_id('id_username') + username_field.send_keys(username) + + password_field = self.selenium.find_element_by_id('id_password1') + password_field.send_keys(password) + password_field = self.selenium.find_element_by_id('id_password2') + password_field.send_keys(password) + + username_field.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def add_user_to_db(self, username, dbname): + database = Database.objects.get(name=dbname, type=self.db_type) + url = self.live_server_url + change_url(database) + self.selenium.get(url) + + user = DatabaseUser.objects.get(username=username, type=self.db_type) + users_from = self.selenium.find_element_by_id('id_users_from') + users_select = Select(users_from) + users_select.select_by_value(str(user.pk)) + + add_user = self.selenium.find_element_by_id('id_users_add_link') + add_user.click() + + save = self.selenium.find_element_by_name('_save') + save.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def swap_user(self, username, username2, dbname): + database = Database.objects.get(name=dbname, type=self.db_type) + url = self.live_server_url + change_url(database) + self.selenium.get(url) + + # remove user "username" + user = DatabaseUser.objects.get(username=username, type=self.db_type) + users_to = self.selenium.find_element_by_id('id_users_to') + users_select = Select(users_to) + users_select.select_by_value(str(user.pk)) + remove_user = self.selenium.find_element_by_id('id_users_remove_link') + remove_user.click() + time.sleep(0.2) + + # add user "username2" + user = DatabaseUser.objects.get(username=username2, type=self.db_type) + users_from = self.selenium.find_element_by_id('id_users_from') + users_select = Select(users_from) + users_select.select_by_value(str(user.pk)) + add_user = self.selenium.find_element_by_id('id_users_add_link') + add_user.click() + time.sleep(0.2) + + save = self.selenium.find_element_by_name('_save') + save.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def delete_user(self, username): + user = DatabaseUser.objects.get(username=username) + self.admin_delete(user) + + +class RESTMysqlDatabaseTest(MySQLControllerMixin, RESTDatabaseMixin, BaseLiveServerTestCase): + pass + + +class AdminMysqlDatabaseTest(MySQLControllerMixin, AdminDatabaseMixin, BaseLiveServerTestCase): + pass diff --git a/orchestra/contrib/domains/__init__.py b/orchestra/contrib/domains/__init__.py new file mode 100644 index 00000000..5c85353e --- /dev/null +++ b/orchestra/contrib/domains/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.domains.apps.DomainsConfig' diff --git a/orchestra/contrib/domains/actions.py b/orchestra/contrib/domains/actions.py new file mode 100644 index 00000000..0e1b222f --- /dev/null +++ b/orchestra/contrib/domains/actions.py @@ -0,0 +1,152 @@ +import copy + +from django.contrib import messages +from django.contrib.admin import helpers +from django.db.models import Q +from django.db.models.functions import Concat, Coalesce +from django.forms.models import modelformset_factory +from django.shortcuts import render +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext_lazy as _ +from django.template.response import TemplateResponse + +from orchestra.admin.utils import get_object_from_url, change_url, admin_link +from orchestra.utils.python import AttrDict + +from .forms import RecordForm, RecordEditFormSet, SOAForm +from .models import Record + + +def view_zone(modeladmin, request, queryset): + zone = queryset.get() + context = { + 'opts': modeladmin.model._meta, + 'object': zone, + 'title': _("%s zone content") % zone.origin.name + } + return TemplateResponse(request, 'admin/domains/domain/view_zone.html', context) +view_zone.url_name = 'view-zone' +view_zone.short_description = _("View zone") + + +def edit_records(modeladmin, request, queryset): + selected_ids = queryset.values_list('id', flat=True) + # Include subodmains + queryset = queryset.model.objects.filter( + Q(top__id__in=selected_ids) | Q(id__in=selected_ids) + ).annotate( + structured_id=Coalesce('top__id', 'id'), + structured_name=Concat('top__name', 'name') + ).order_by('-structured_id', 'structured_name') + formsets = [] + for domain in queryset.prefetch_related('records'): + modeladmin_copy = copy.copy(modeladmin) + modeladmin_copy.model = Record + prefix = '' if domain.is_top else ' '*8 + context = { + 'url': change_url(domain), + 'name': prefix+domain.name, + 'title': '', + } + if domain.id not in selected_ids: + context['name'] += '*' + context['title'] = _("This subdomain was not explicitly selected " + "but has been automatically added to this list.") + link = '%(name)s' % context + modeladmin_copy.verbose_name_plural = mark_safe(link) + RecordFormSet = modelformset_factory( + Record, form=RecordForm, formset=RecordEditFormSet, extra=1, can_delete=True) + formset = RecordFormSet(queryset=domain.records.all(), prefix=domain.id) + formset.instance = domain + formset.cls = RecordFormSet + formsets.append(formset) + + if request.POST.get('post') == 'generic_confirmation': + posted_formsets = [] + all_valid = True + for formset in formsets: + instance = formset.instance + formset = formset.cls( + request.POST, request.FILES, queryset=formset.queryset, prefix=instance.id) + formset.instance = instance + if not formset.is_valid(): + all_valid = False + posted_formsets.append(formset) + formsets = posted_formsets + if all_valid: + for formset in formsets: + for form in formset.forms: + form.instance.domain_id = formset.instance.id + formset.save() + fake_form = AttrDict({ + 'changed_data': False + }) + change_message = modeladmin.construct_change_message(request, fake_form, [formset]) + modeladmin.log_change(request, formset.instance, change_message) + num = len(formsets) + message = ngettext( + _("Records for one selected domain have been updated."), + _("Records for %i selected domains have been updated.") % num, + num) + modeladmin.message_user(request, message) + return + + opts = modeladmin.model._meta + context = { + 'title': _("Edit records"), + 'action_name': _("Edit records"), + 'action_value': 'edit_records', + 'display_objects': [], + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'formsets': formsets, + 'obj': get_object_from_url(modeladmin, request), + } + return render(request, 'admin/domains/domain/edit_records.html', context) + + +def set_soa(modeladmin, request, queryset): + if queryset.filter(top__isnull=False).exists(): + msg = _("Set SOA on subdomains is not possible.") + modeladmin.message_user(request, msg, messages.ERROR) + return + form = SOAForm() + if request.POST.get('post') == 'generic_confirmation': + form = SOAForm(request.POST) + if form.is_valid(): + updates = {name: value for name, value in form.cleaned_data.items() if value} + change_message = _("SOA set %s") % str(updates)[1:-1] + for domain in queryset: + for name, value in updates.items(): + if name.startswith('clear_'): + name = name.replace('clear_', '') + value = '' + setattr(domain, name, value) + modeladmin.log_change(request, domain, change_message) + domain.save() + num = len(queryset) + msg = ngettext( + _("SOA record for one domain has been updated."), + _("SOA record for %s domains has been updated.") % num, + num + ) + modeladmin.message_user(request, msg) + return + opts = modeladmin.model._meta + context = { + 'title': _("Set SOA for selected domains"), + 'content_message': '', + 'action_name': _("Set SOA"), + 'action_value': 'set_soa', + 'display_objects': [admin_link('__str__')(domain) for domain in queryset], + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'form': form, + 'obj': get_object_from_url(modeladmin, request), + } + return render(request, 'admin/orchestra/generic_confirmation.html', context) +set_soa.short_description = _("Set SOA for selected domains") diff --git a/orchestra/contrib/domains/admin.py b/orchestra/contrib/domains/admin.py new file mode 100644 index 00000000..10994cf4 --- /dev/null +++ b/orchestra/contrib/domains/admin.py @@ -0,0 +1,227 @@ +from django.contrib import admin +from django.urls import reverse +from django.db import models +from django.db.models.functions import Concat, Coalesce +from django.templatetags.static import static +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext, gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import admin_link, change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.utils import apps +from orchestra.utils.html import get_on_site_link + +from . import settings +from .actions import view_zone, edit_records, set_soa +from .filters import TopDomainListFilter, HasWebsiteFilter, HasAddressFilter +from .forms import RecordForm, RecordInlineFormSet, BatchDomainCreationAdminForm +from .models import Domain, Record + + +class RecordInline(admin.TabularInline): + model = Record + form = RecordForm + formset = RecordInlineFormSet + verbose_name_plural = _("Extra records") + + +class DomainInline(admin.TabularInline): + model = Domain + fields = ('domain_link', 'display_records', 'account_link') + readonly_fields = ('domain_link', 'display_records', 'account_link') + extra = 0 + verbose_name_plural = _("Subdomains") + + domain_link = admin_link('__str__') + domain_link.short_description = _("Name") + account_link = admin_link('account') + + def display_records(self, domain): + return ', '.join([record.type for record in domain.records.all()]) + display_records.short_description = _("Declared records") + + def has_add_permission(self, *args, **kwargs): + return False + + def get_queryset(self, request): + """ Order by structured name and imporve performance """ + qs = super(DomainInline, self).get_queryset(request) + return qs.select_related('account').prefetch_related('records') + + +class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'structured_name', 'display_is_top', 'display_websites', 'display_addresses', 'account_link' + ) + add_fields = ('name', 'account') + fields = ('name', 'account_link', 'display_websites', 'display_addresses', 'dns2136_address_match_list') + readonly_fields = ( + 'account_link', 'top_link', 'display_websites', 'display_addresses', 'implicit_records' + ) + inlines = (RecordInline, DomainInline) + list_filter = (TopDomainListFilter, HasWebsiteFilter, HasAddressFilter) + change_readonly_fields = ('name', 'serial') + search_fields = ('name', 'account__username', 'records__value') + add_form = BatchDomainCreationAdminForm + actions = (edit_records, set_soa, list_accounts) + change_view_actions = (view_zone, edit_records) + + top_link = admin_link('top') + + def structured_name(self, domain): + if domain.is_top: + return domain.name + return mark_safe(' '*4 + domain.name) + structured_name.short_description = _("name") + structured_name.admin_order_field = 'structured_name' + + def display_is_top(self, domain): + return domain.is_top + display_is_top.short_description = _("Is top") + display_is_top.boolean = True + display_is_top.admin_order_field = 'top' + + @mark_safe + def display_websites(self, domain): + if apps.isinstalled('orchestra.contrib.websites'): + websites = domain.websites.all() + if websites: + links = [] + for website in websites: + site_link = get_on_site_link(website.get_absolute_url()) + admin_url = change_url(website) + title = _("Edit website") + link = format_html('{} {}', + admin_url, title, website.name, site_link) + links.append(link) + return '
    '.join(links) + add_url = reverse('admin:websites_website_add') + add_url += '?account=%i&domains=%i' % (domain.account_id, domain.pk) + add_link = format_html( + '', add_url, + _("Add website"), static('orchestra/images/add.png'), + ) + return _("No website %s") % (add_link) + return '---' + display_websites.admin_order_field = 'websites__name' + display_websites.short_description = _("Websites") + + @mark_safe + def display_addresses(self, domain): + if apps.isinstalled('orchestra.contrib.mailboxes'): + add_url = reverse('admin:mailboxes_address_add') + add_url += '?account=%i&domain=%i' % (domain.account_id, domain.pk) + image = '' % static('orchestra/images/add.png') + add_link = '%s' % ( + add_url, _("Add address"), image + ) + addresses = domain.addresses.all() + if addresses: + url = reverse('admin:mailboxes_address_changelist') + url += '?domain=%i' % addresses[0].domain_id + title = '\n'.join([address.email for address in addresses]) + return '%s %s' % (url, title, len(addresses), add_link) + return _("No address %s") % (add_link) + return '---' + display_addresses.short_description = _("Addresses") + display_addresses.admin_order_field = 'addresses__count' + + @mark_safe + def implicit_records(self, domain): + types = set(domain.records.values_list('type', flat=True)) + ttl = settings.DOMAINS_DEFAULT_TTL + lines = [] + for record in domain.get_default_records(): + line = '{name} {ttl} IN {type} {value}'.format( + name=domain.name, + ttl=ttl, + type=record.type, + value=record.value + ) + if not domain.record_is_implicit(record, types): + line = format_html('{}', line) + if record.type is Record.SOA: + lines.insert(0, line) + else: + lines.append(line) + return '
    '.join(lines) + implicit_records.short_description = _("Implicit records") + + def get_fieldsets(self, request, obj=None): + """ Add SOA fields when domain is top """ + fieldsets = super(DomainAdmin, self).get_fieldsets(request, obj) + if obj: + fieldsets += ( + (_("Implicit records"), { + 'classes': ('collapse',), + 'fields': ('implicit_records',), + }), + ) + if obj.is_top: + fieldsets += ( + (_("SOA"), { + 'classes': ('collapse',), + 'description': _( + "SOA (Start of Authority) records are used to determine how the " + "zone propagates to the secondary nameservers."), + 'fields': ('serial', 'refresh', 'retry', 'expire', 'min_ttl'), + }), + ) + else: + existing = fieldsets[0][1]['fields'] + if 'top_link' not in existing: + fieldsets[0][1]['fields'].insert(2, 'top_link') + return fieldsets + + def get_inline_instances(self, request, obj=None): + inlines = super(DomainAdmin, self).get_inline_instances(request, obj) + if not obj or not obj.is_top: + return [inline for inline in inlines if type(inline) != DomainInline] + return inlines + + def get_queryset(self, request): + """ Order by structured name and imporve performance """ + qs = super(DomainAdmin, self).get_queryset(request) + qs = qs.select_related('top', 'account') + if request.method == 'GET': + qs = qs.annotate( + structured_id=Coalesce('top__id', 'id'), + structured_name=Concat('top__name', 'name') + ).order_by('-structured_id', 'structured_name') + if apps.isinstalled('orchestra.contrib.websites'): + qs = qs.prefetch_related('websites__domains') + if apps.isinstalled('orchestra.contrib.mailboxes'): + qs = qs.annotate(models.Count('addresses')) + return qs + + def save_model(self, request, obj, form, change): + """ batch domain creation support """ + super(DomainAdmin, self).save_model(request, obj, form, change) + self.extra_domains = [] + if not change: + for name in form.extra_names: + domain = Domain.objects.create(name=name, account_id=obj.account_id) + self.extra_domains.append(domain) + + def save_related(self, request, form, formsets, change): + """ batch domain creation support """ + super(DomainAdmin, self).save_related(request, form, formsets, change) + if not change: + # Clone records to extra_domains, if any + for formset in formsets: + if formset.model is Record: + for domain in self.extra_domains: + # Reset pk value of the record instances to force creation of new ones + for record_form in formset.forms: + record = record_form.instance + if record.pk: + record.pk = None + formset.instance = domain + form.instance = domain + self.save_formset(request, form, formset, change) + + +admin.site.register(Domain, DomainAdmin) diff --git a/orchestra/contrib/domains/api.py b/orchestra/contrib/domains/api.py new file mode 100644 index 00000000..27a25453 --- /dev/null +++ b/orchestra/contrib/domains/api.py @@ -0,0 +1,38 @@ +from rest_framework import viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from orchestra.api import router +from orchestra.contrib.accounts.api import AccountApiMixin + +from . import settings +from .models import Domain +from .serializers import DomainSerializer + + +class DomainViewSet(AccountApiMixin, viewsets.ModelViewSet): + serializer_class = DomainSerializer + filter_fields = ('name',) + queryset = Domain.objects.all() + + def get_queryset(self): + qs = super(DomainViewSet, self).get_queryset() + return qs.prefetch_related('records') + + @action(detail=True) + def view_zone(self, request, pk=None): + domain = self.get_object() + return Response({ + 'zone': domain.render_zone() + }) + + def options(self, request): + metadata = super(DomainViewSet, self).options(request) + names = ['DOMAINS_DEFAULT_A', 'DOMAINS_DEFAULT_MX', 'DOMAINS_DEFAULT_NS'] + metadata.data['settings'] = { + name.lower(): getattr(settings, name, None) for name in names + } + return metadata + + +router.register(r'domains', DomainViewSet) diff --git a/orchestra/contrib/domains/apps.py b/orchestra/contrib/domains/apps.py new file mode 100644 index 00000000..559166c4 --- /dev/null +++ b/orchestra/contrib/domains/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class DomainsConfig(AppConfig): + name = 'orchestra.contrib.domains' + verbose_name = 'Domains' + + def ready(self): + from .models import Domain + services.register(Domain, icon='domain.png') diff --git a/orchestra/contrib/domains/backends.py b/orchestra/contrib/domains/backends.py new file mode 100644 index 00000000..e160e61b --- /dev/null +++ b/orchestra/contrib/domains/backends.py @@ -0,0 +1,224 @@ +import re +import socket +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import Operation +from orchestra.utils.python import OrderedSet + +from . import settings +from .models import Record, Domain + + +class Bind9MasterDomainController(ServiceController): + """ + Bind9 zone and config generation. + It auto-discovers slave Bind9 servers based on your routing configuration and NS servers. + """ + CONF_PATH = settings.DOMAINS_MASTERS_PATH + + verbose_name = _("Bind9 master domain") + model = 'domains.Domain' + related_models = ( + ('domains.Record', 'domain__origin'), + ('domains.Domain', 'origin'), + ) + ignore_fields = ('serial',) + doc_settings = (settings, + ('DOMAINS_MASTERS_PATH',) + ) + + @classmethod + def is_main(cls, obj): + """ work around Domain.top self relationship """ + if super(Bind9MasterDomainController, cls).is_main(obj): + return not obj.top + + def save(self, domain): + context = self.get_context(domain) + domain.refresh_serial() + self.update_zone(domain, context) + self.update_conf(context) + + def update_zone(self, domain, context): + context['zone'] = ';; %(banner)s\n' % context + context['zone'] += domain.render_zone() + self.append(textwrap.dedent("""\ + # Generate %(name)s zone file + cat << 'EOF' > %(zone_path)s.tmp + %(zone)s + EOF + diff -N -I'^\s*;;' %(zone_path)s %(zone_path)s.tmp || UPDATED=1 + # Because bind reload will not display any fucking error + named-checkzone -k fail -n fail %(name)s %(zone_path)s.tmp + mv %(zone_path)s.tmp %(zone_path)s\ + """) % context + ) + + def update_conf(self, context): + self.append(textwrap.dedent(""" + # Update bind config file for %(name)s + read -r -d '' conf << 'EOF' || true + %(conf)s + EOF + sed '/zone "%(name)s".*/,/^\s*};\s*$/!d' %(conf_path)s | diff -B -I"^\s*//" - <(echo "${conf}") || { + sed -i -e '/zone\s\s*"%(name)s".*/,/^\s*};/d' \\ + -e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s + echo "${conf}" >> %(conf_path)s + UPDATED=1 + }""") % context + ) + self.append(textwrap.dedent("""\ + # Delete ex-top-domains that are now subdomains + sed -i -e '/zone\s\s*".*\.%(name)s".*/,/^\s*};\s*$/d' \\ + -e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s""") % context + ) + if 'zone_path' in context: + context['zone_subdomains_path'] = re.sub(r'^(.*/)', r'\1*.', context['zone_path']) + self.append('rm -f -- %(zone_subdomains_path)s' % context) + + def delete(self, domain): + context = self.get_context(domain) + self.append('# Delete zone file for %(name)s' % context) + self.append('rm -f -- %(zone_path)s;' % context) + self.delete_conf(context) + + def delete_conf(self, context): + if context['name'][0] in ('*', '_'): + # These can never be top level domains + return + self.append(textwrap.dedent(""" + # Delete config for %(name)s + sed -e '/zone\s\s*"%(name)s".*/,/^\s*};\s*$/d' \\ + -e 'N; /^\s*\\n\s*$/d; P; D' %(conf_path)s > %(conf_path)s.tmp""") % context + ) + self.append('diff -B -I"^\s*//" %(conf_path)s.tmp %(conf_path)s || UPDATED=1' % context) + self.append('mv %(conf_path)s.tmp %(conf_path)s' % context) + + def commit(self): + """ reload bind if needed """ + self.append(textwrap.dedent(""" + # Apply changes + if [[ $UPDATED == 1 ]]; then + rm /etc/bind/master/*jnl || true; service bind9 restart + fi""") + ) + + def get_servers(self, domain, backend): + """ Get related server IPs from registered backend routes """ + from orchestra.contrib.orchestration.manager import router + operation = Operation(backend, domain, Operation.SAVE) + servers = [] + for route in router.objects.get_for_operation(operation): + servers.append(route.host.get_ip()) + return servers + + def get_masters_ips(self, domain): + ips = list(settings.DOMAINS_MASTERS) + if not ips: + ips += self.get_servers(domain, Bind9MasterDomainController) + return OrderedSet(sorted(ips)) + + def get_slaves(self, domain): + ips = [] + masters_ips = self.get_masters_ips(domain) + records = domain.get_records() + # Slaves from NS + for record in records.by_type(Record.NS): + hostname = record.value.rstrip('.') + # First try with a DNS query, a more reliable source + try: + addr = socket.gethostbyname(hostname) + except socket.gaierror: + # check if hostname is declared + try: + domain = Domain.objects.get(name=hostname) + except Domain.DoesNotExist: + continue + else: + # default to domain A record address + addr = records.by_type(Record.A)[0].value + if addr not in masters_ips: + ips.append(addr) + # Slaves from internal networks + if not settings.DOMAINS_MASTERS: + for server in self.get_servers(domain, Bind9SlaveDomainController): + ips.append(server) + return OrderedSet(sorted(ips)) + + def get_context(self, domain): + slaves = self.get_slaves(domain) + context = { + 'name': domain.name, + 'zone_path': settings.DOMAINS_ZONE_PATH % {'name': domain.name}, + 'subdomains': domain.subdomains.all(), + 'banner': self.get_banner(), + 'slaves': '; '.join(slaves) or 'none', + 'also_notify': '; '.join(slaves) + ';' if slaves else '', + 'conf_path': self.CONF_PATH, + 'dns2136_address_match_list': domain.dns2136_address_match_list + } + context['conf'] = textwrap.dedent("""\ + zone "%(name)s" { + // %(banner)s + type master; + file "%(zone_path)s"; + allow-transfer { %(slaves)s; }; + also-notify { %(also_notify)s }; + allow-update { %(dns2136_address_match_list)s }; + notify yes; + };""") % context + return context + + +class Bind9SlaveDomainController(Bind9MasterDomainController): + """ + Generate the configuartion for slave servers + It auto-discover the master server based on your routing configuration or you can use + DOMAINS_MASTERS to explicitly configure the master. + """ + CONF_PATH = settings.DOMAINS_SLAVES_PATH + + verbose_name = _("Bind9 slave domain") + related_models = ( + ('domains.Domain', 'origin'), + ) + doc_settings = (settings, + ('DOMAINS_MASTERS', 'DOMAINS_SLAVES_PATH') + ) + def save(self, domain): + context = self.get_context(domain) + self.update_conf(context) + + def delete(self, domain): + context = self.get_context(domain) + self.delete_conf(context) + + def commit(self): + self.append(textwrap.dedent(""" + # Apply changes + if [[ $UPDATED == 1 ]]; then + # Async restart, ideally after master + nohup bash -c 'sleep 1 && service bind9 reload' &> /dev/null & + fi""") + ) + + def get_context(self, domain): + context = { + 'name': domain.name, + 'banner': self.get_banner(), + 'subdomains': domain.subdomains.all(), + 'masters': '; '.join(self.get_masters_ips(domain)) or 'none', + 'conf_path': self.CONF_PATH, + } + context['conf'] = textwrap.dedent("""\ + zone "%(name)s" { + // %(banner)s + type slave; + file "%(name)s"; + masters { %(masters)s; }; + allow-notify { %(masters)s; }; + };""") % context + return context diff --git a/orchestra/contrib/domains/filters.py b/orchestra/contrib/domains/filters.py new file mode 100644 index 00000000..c12f48e2 --- /dev/null +++ b/orchestra/contrib/domains/filters.py @@ -0,0 +1,49 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + + +class TopDomainListFilter(SimpleListFilter): + """ Filter Nodes by group according to request.user """ + title = _("top domains") + parameter_name = 'top_domain' + + def lookups(self, request, model_admin): + return ( + ('True', _("Top domains")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(top__isnull=True) + + +class HasWebsiteFilter(SimpleListFilter): + """ Filter addresses whether they have any websites or not """ + title = _("has websites") + parameter_name = 'has_websites' + + def lookups(self, request, model_admin): + return ( + ('True', _("True")), + ('False', _("False")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(websites__isnull=False) + elif self.value() == 'False': + return queryset.filter(websites__isnull=True) + return queryset + + +class HasAddressFilter(HasWebsiteFilter): + """ Filter addresses whether they have any addresses or not """ + title = _("has addresses") + parameter_name = 'has_addresses' + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(addresses__isnull=False) + elif self.value() == 'False': + return queryset.filter(addresses__isnull=True) + return queryset diff --git a/orchestra/contrib/domains/forms.py b/orchestra/contrib/domains/forms.py new file mode 100644 index 00000000..15db21c8 --- /dev/null +++ b/orchestra/contrib/domains/forms.py @@ -0,0 +1,164 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.utils.text import capfirst +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin.forms import AdminFormSet, AdminFormMixin + +from . import validators +from .helpers import domain_for_validation +from .models import Domain, Record + + +class BatchDomainCreationAdminForm(forms.ModelForm): + name = forms.CharField(label=_("Names"), widget=forms.Textarea(attrs={'rows': 5, 'cols': 50}), + help_text=_("Fully qualified domain name per line. " + "All domains will have the provided account and records.")) + + def clean_name(self): + self.extra_names = [] + target = None + existing = set() + errors = [] + domain_names = self.cleaned_data['name'].strip().splitlines() + for name in domain_names: + name = name.strip() + if not name: + continue + if name in existing: + errors.append(ValidationError(_("%s domain name provided multiple times.") % name)) + existing.add(name) + if target is None: + target = name + else: + domain = Domain(name=name) + try: + domain.full_clean(exclude=['top']) + except ValidationError as e: + for error in e.error_dict['name']: + for msg in error.messages: + errors.append( + ValidationError("%s: %s" % (name, msg)) + ) + self.extra_names.append(name) + if errors: + raise ValidationError(errors) + return target + + def clean(self): + """ inherit related parent domain account, when exists """ + cleaned_data = super().clean() + if not cleaned_data['account']: + account = None + domain_names = [] + if 'name' in cleaned_data: + first = cleaned_data['name'] + domain_names.append(first) + domain_names.extend(self.extra_names) + for name in domain_names: + parent = Domain.objects.get_parent(name) + if not parent: + # Fake an account to make django validation happy + account_model = self.fields['account']._queryset.model + cleaned_data['account'] = account_model() + raise ValidationError({ + 'account': _("An account should be provided for top domain names."), + }) + elif account and parent.account != account: + # Fake an account to make django validation happy + account_model = self.fields['account']._queryset.model + cleaned_data['account'] = account_model() + raise ValidationError({ + 'account': _("Provided domain names belong to different accounts."), + }) + account = parent.account + cleaned_data['account'] = account + return cleaned_data + + def full_clean(self): + # set extra_names on instance to use it on inline formsets validation + super().full_clean() + self.instance.extra_names = extra_names + + +class RecordForm(forms.ModelForm): + class Meta: + fields = ('ttl', 'type', 'value') + + def __init__(self, *args, **kwargs): + super(RecordForm, self).__init__(*args, **kwargs) + self.fields['ttl'].widget = forms.TextInput(attrs={'size': '10'}) + self.fields['value'].widget = forms.TextInput(attrs={'size': '100'}) + + +class ValidateZoneMixin(object): + def clean(self): + """ Checks if everything is consistent """ + super(ValidateZoneMixin, self).clean() + if any(self.errors): + return + is_host = True + for form in self.forms: + if form.cleaned_data.get('type') in (Record.TXT, Record.SRV, Record.CNAME): + is_host = False + break + domain_names = [] + if self.instance.name: + domain_names.append(self.instance.name) + domain_names.extend(getattr(self.instance, 'extra_names', [])) + errors = [] + for name in domain_names: + records = [] + for form in self.forms: + data = form.cleaned_data + if data and not data['DELETE']: + records.append(data) + if '_' in name and is_host: + errors.append(ValidationError( + _("%s: Hosts can not have underscore character '_', consider providing a SRV, CNAME or TXT record.") % name + )) + domain = domain_for_validation(self.instance, records) + try: + validators.validate_zone(domain.render_zone()) + except ValidationError as error: + for msg in error: + errors.append( + ValidationError("%s: %s" % (name, msg)) + ) + if errors: + raise ValidationError(errors) + + +class RecordEditFormSet(ValidateZoneMixin, AdminFormSet): + pass + + +class RecordInlineFormSet(ValidateZoneMixin, forms.models.BaseInlineFormSet): + pass + + +class SOAForm(AdminFormMixin, forms.Form): + refresh = forms.CharField() + clear_refresh = forms.BooleanField(label=_("Clear refresh"), required=False, + help_text=_("Remove custom refresh value for all selected domains.")) + retry = forms.CharField() + clear_retry = forms.BooleanField(label=_("Clear retry"), required=False, + help_text=_("Remove custom retry value for all selected domains.")) + expire = forms.CharField() + clear_expire = forms.BooleanField(label=_("Clear expire"), required=False, + help_text=_("Remove custom expire value for all selected domains.")) + min_ttl = forms.CharField() + clear_min_ttl = forms.BooleanField(label=_("Clear min TTL"), required=False, + help_text=_("Remove custom min TTL value for all selected domains.")) + + def __init__(self, *args, **kwargs): + super(SOAForm, self).__init__(*args, **kwargs) + for name in self.fields: + if not name.startswith('clear_'): + field = Domain._meta.get_field(name) + self.fields[name] = forms.CharField( + label=capfirst(field.verbose_name), + help_text=field.help_text, + validators=field.validators, + required=False, + ) diff --git a/orchestra/contrib/domains/helpers.py b/orchestra/contrib/domains/helpers.py new file mode 100644 index 00000000..a161fb84 --- /dev/null +++ b/orchestra/contrib/domains/helpers.py @@ -0,0 +1,32 @@ +import copy + +from .models import Domain, Record + + +def domain_for_validation(instance, records): + """ + Since the new data is not yet on the database, we update it on the fly, + so when validation calls render_zone() it will use the new provided data + """ + domain = copy.copy(instance) + def get_declared_records(records=records): + for data in records: + yield Record(type=data['type'], value=data['value']) + domain.get_declared_records = get_declared_records + + if not domain.pk: + # top domain lookup for new domains + domain.top = domain.get_parent(top=True) + if domain.top: + # is a subdomain + subdomains = domain.top.subdomains.select_related('top').prefetch_related('records').all() + subdomains = [sub for sub in subdomains if sub.pk != domain.pk] + domain.top.get_subdomains = lambda: subdomains + [domain] + elif not domain.pk: + # is a new top domain + subdomains = [] + for subdomain in Domain.objects.filter(name__endswith='.%s' % domain.name): + subdomain.top = domain + subdomains.append(subdomain) + domain.get_subdomains = lambda: subdomains + return domain diff --git a/orchestra/contrib/domains/models.py b/orchestra/contrib/domains/models.py new file mode 100644 index 00000000..9d099aa4 --- /dev/null +++ b/orchestra/contrib/domains/models.py @@ -0,0 +1,352 @@ +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.core.validators import validate_ipv4_address, validate_ipv6_address, validate_ascii +from orchestra.utils.python import AttrDict + +from . import settings, validators, utils + + +class DomainQuerySet(models.QuerySet): + def get_parent(self, name, top=False): + """ get the next domain on the chain """ + split = name.split('.') + parent = None + for i in range(1, len(split)-1): + name = '.'.join(split[i:]) + domain = Domain.objects.filter(name=name) + if domain: + parent = domain.get() + if not top: + return parent + return parent + + +class Domain(models.Model): + name = models.CharField(_("name"), max_length=256, unique=True, db_index=True, + help_text=_("Domain or subdomain name."), + validators=[ + validators.validate_domain_name, + validators.validate_allowed_domain + ]) + account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), blank=True, + related_name='domains', on_delete=models.CASCADE, help_text=_("Automatically selected for subdomains.")) + top = models.ForeignKey('domains.Domain', null=True, related_name='subdomain_set', + editable=False, verbose_name=_("top domain"), on_delete=models.CASCADE) + serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, editable=False, + help_text=_("A revision number that changes whenever this domain is updated.")) + refresh = models.CharField(_("refresh"), max_length=16, blank=True, + validators=[validators.validate_zone_interval], + help_text=_("The time a secondary DNS server waits before querying the primary DNS " + "server's SOA record to check for changes. When the refresh time expires, " + "the secondary DNS server requests a copy of the current SOA record from " + "the primary. The primary DNS server complies with this request. " + "The secondary DNS server compares the serial number of the primary DNS " + "server's current SOA record and the serial number in it's own SOA record. " + "If they are different, the secondary DNS server will request a zone " + "transfer from the primary DNS server. " + "The default value is %s.") % settings.DOMAINS_DEFAULT_REFRESH) + retry = models.CharField(_("retry"), max_length=16, blank=True, + validators=[validators.validate_zone_interval], + help_text=_("The time a secondary server waits before retrying a failed zone transfer. " + "Normally, the retry time is less than the refresh time. " + "The default value is %s.") % settings.DOMAINS_DEFAULT_RETRY) + expire = models.CharField(_("expire"), max_length=16, blank=True, + validators=[validators.validate_zone_interval], + help_text=_("The time that a secondary server will keep trying to complete a zone " + "transfer. If this time expires prior to a successful zone transfer, " + "the secondary server will expire its zone file. This means the secondary " + "will stop answering queries. " + "The default value is %s.") % settings.DOMAINS_DEFAULT_EXPIRE) + min_ttl = models.CharField(_("min TTL"), max_length=16, blank=True, + validators=[validators.validate_zone_interval], + help_text=_("The minimum time-to-live value applies to all resource records in the " + "zone file. This value is supplied in query responses to inform other " + "servers how long they should keep the data in cache. " + "The default value is %s.") % settings.DOMAINS_DEFAULT_MIN_TTL) + dns2136_address_match_list = models.CharField(max_length=80, default=settings.DOMAINS_DEFAULT_DNS2136, + blank=True, + help_text="A bind-9 'address_match_list' that will be granted permission to perform " + "dns2136 updates. Chiefly used to enable Let's Encrypt self-service validation.") + + objects = DomainQuerySet.as_manager() + + def __str__(self): + return self.name + + @property + def origin(self): + return self.top or self + + @property + def is_top(self): + # don't cache, don't replace by top_id + try: + return not bool(self.top) + except Domain.DoesNotExist: + return False + + @property + def subdomains(self): + return Domain.objects.filter(name__regex='\.%s$' % self.name) + + def clean(self): + self.name = self.name.lower() + + def save(self, *args, **kwargs): + """ create top relation """ + update = False + if not self.pk: + top = self.get_parent(top=True) + if top: + self.top = top + self.account_id = self.account_id or top.account_id + else: + update = True + super(Domain, self).save(*args, **kwargs) + if update: + for domain in self.subdomains.exclude(pk=self.pk): + # queryset.update() is not used because we want to trigger backend to delete ex-topdomains + domain.top = self + domain.save(update_fields=('top',)) + + def get_description(self): + if self.is_top: + num = self.subdomains.count() + return ngettext( + _("top domain with one subdomain"), + _("top domain with %d subdomains") % num, + num) + return _("subdomain") + + def get_absolute_url(self): + return 'http://%s' % self.name + + def get_declared_records(self): + """ proxy method, needed for input validation, see helpers.domain_for_validation """ + return self.records.all() + + def get_subdomains(self): + """ proxy method, needed for input validation, see helpers.domain_for_validation """ + return self.origin.subdomain_set.all().prefetch_related('records') + + def get_parent(self, top=False): + return type(self).objects.get_parent(self.name, top=top) + + def render_zone(self): + origin = self.origin + zone = origin.render_records() + tail = [] + for subdomain in origin.get_subdomains(): + if subdomain.name.startswith('*'): + # This subdomains needs to be rendered last in order to avoid undesired matches + tail.append(subdomain) + else: + zone += subdomain.render_records() + ###darmengo 2021-03-25 add autoconfig + if self.has_default_mx(): + zone += 'autoconfig.{}. 30m IN A 109.69.8.133\n'.format(self.name) + ###END darmengo 2021-03-25 add autoconfig + for subdomain in sorted(tail, key=lambda x: len(x.name), reverse=True): + zone += subdomain.render_records() + return zone.strip() + + def refresh_serial(self): + """ Increases the domain serial number by one """ + serial = utils.generate_zone_serial() + if serial <= self.serial: + num = int(str(self.serial)[8:]) + 1 + if num >= 99: + raise ValueError('No more serial numbers for today') + serial = str(self.serial)[:8] + '%.2d' % num + serial = int(serial) + self.serial = serial + self.save(update_fields=('serial',)) + + def get_default_soa(self): + return ' '.join([ + "%s." % settings.DOMAINS_DEFAULT_NAME_SERVER, + utils.format_hostmaster(settings.DOMAINS_DEFAULT_HOSTMASTER), + str(self.serial), + self.refresh or settings.DOMAINS_DEFAULT_REFRESH, + self.retry or settings.DOMAINS_DEFAULT_RETRY, + self.expire or settings.DOMAINS_DEFAULT_EXPIRE, + self.min_ttl or settings.DOMAINS_DEFAULT_MIN_TTL, + ]) + + def get_default_records(self): + defaults = [] + if self.is_top: + for ns in settings.DOMAINS_DEFAULT_NS: + defaults.append(AttrDict( + type=Record.NS, + value=ns + )) + soa = self.get_default_soa() + defaults.insert(0, AttrDict( + type=Record.SOA, + value=soa + )) + for mx in settings.DOMAINS_DEFAULT_MX: + defaults.append(AttrDict( + type=Record.MX, + value=mx + )) + default_a = settings.DOMAINS_DEFAULT_A + if default_a: + defaults.append(AttrDict( + type=Record.A, + value=default_a + )) + default_aaaa = settings.DOMAINS_DEFAULT_AAAA + if default_aaaa: + defaults.append(AttrDict( + type=Record.AAAA, + value=default_aaaa + )) + return defaults + + def record_is_implicit(self, record, types): + if record.type not in types: + if record.type is Record.NS: + if self.is_top: + return True + elif record.type is Record.SOA: + if self.is_top: + return True + else: + has_a = Record.A in types + has_aaaa = Record.AAAA in types + is_host = self.is_top or not types or has_a or has_aaaa + if is_host: + if record.type is Record.MX: + return True + elif not has_a and not has_aaaa: + return True + return False + + def get_records(self): + types = set() + records = utils.RecordStorage() + for record in self.get_declared_records(): + types.add(record.type) + if record.type == record.SOA: + # Update serial and insert at 0 + value = record.value.split() + value[2] = str(self.serial) + records.insert(0, AttrDict( + type=record.SOA, + ttl=record.get_ttl(), + value=' '.join(value) + )) + else: + records.append(AttrDict( + type=record.type, + ttl=record.get_ttl(), + value=record.value + )) + for record in self.get_default_records(): + if self.record_is_implicit(record, types): + if record.type is Record.SOA: + records.insert(0, record) + else: + records.append(record) + return records + + def render_records(self): + result = '' + for record in self.get_records(): + name = '{name}.{spaces}'.format( + name=self.name, + spaces=' ' * (37-len(self.name)) + ) + ttl = record.get('ttl', settings.DOMAINS_DEFAULT_TTL) + ttl = '{spaces}{ttl}'.format( + spaces=' ' * (7-len(ttl)), + ttl=ttl + ) + type = '{type} {spaces}'.format( + type=record.type, + spaces=' ' * (7-len(record.type)) + ) + result += '{name} {ttl} IN {type} {value}\n'.format( + name=name, + ttl=ttl, + type=type, + value=record.value + ) + return result + + def has_default_mx(self): + records = self.get_records() + for record in records.by_type('MX'): + for default in settings.DOMAINS_DEFAULT_MX: + if record.value.endswith(' %s' % default.split()[-1]): + return True + return False + + +class Record(models.Model): + """ Represents a domain resource record """ + MX = 'MX' + NS = 'NS' + CNAME = 'CNAME' + A = 'A' + AAAA = 'AAAA' + SRV = 'SRV' + TXT = 'TXT' + SPF = 'SPF' + SOA = 'SOA' + + TYPE_CHOICES = ( + (MX, "MX"), + (NS, "NS"), + (CNAME, "CNAME"), + (A, _("A (IPv4 address)")), + (AAAA, _("AAAA (IPv6 address)")), + (SRV, "SRV"), + (TXT, "TXT"), + (SPF, "SPF"), + ) + + VALIDATORS = { + MX: (validators.validate_mx_record,), + NS: (validators.validate_zone_label,), + A: (validate_ipv4_address,), + AAAA: (validate_ipv6_address,), + CNAME: (validators.validate_zone_label,), + TXT: (validate_ascii, validators.validate_quoted_record), + SPF: (validate_ascii, validators.validate_quoted_record), + SRV: (validators.validate_srv_record,), + SOA: (validators.validate_soa_record,), + } + + domain = models.ForeignKey(Domain, verbose_name=_("domain"), related_name='records', on_delete=models.CASCADE) + ttl = models.CharField(_("TTL"), max_length=8, blank=True, + help_text=_("Record TTL, defaults to %s") % settings.DOMAINS_DEFAULT_TTL, + validators=[validators.validate_zone_interval]) + type = models.CharField(_("type"), max_length=32, choices=TYPE_CHOICES) + # max_length bumped from 256 to 1024 (arbitrary) on August 2019. + value = models.CharField(_("value"), max_length=1024, + help_text=_("MX, NS and CNAME records sould end with a dot.")) + + def __str__(self): + return "%s %s IN %s %s" % (self.domain, self.get_ttl(), self.type, self.value) + + def clean(self): + """ validates record value based on its type """ + # validate value + if self.type != self.TXT: + self.value = self.value.lower().strip() + if self.type: + for validator in self.VALIDATORS.get(self.type, []): + try: + validator(self.value) + except ValidationError as error: + raise ValidationError({ + 'value': error, + }) + + def get_ttl(self): + return self.ttl or settings.DOMAINS_DEFAULT_TTL diff --git a/orchestra/contrib/domains/serializers.py b/orchestra/contrib/domains/serializers.py new file mode 100644 index 00000000..7c6bebd6 --- /dev/null +++ b/orchestra/contrib/domains/serializers.py @@ -0,0 +1,82 @@ +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.api.serializers import HyperlinkedModelSerializer +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .helpers import domain_for_validation +from .models import Domain, Record +from . import validators + + +class RecordSerializer(serializers.ModelSerializer): + class Meta: + model = Record + fields = ('type', 'value') + + def get_identity(self, data): + return data.get('value') + + +class DomainSerializer(AccountSerializerMixin, HyperlinkedModelSerializer): + """ Validates if this zone generates a correct zone file """ + records = RecordSerializer(required=False, many=True) + + class Meta: + model = Domain + fields = ('url', 'id', 'name', 'records') + postonly_fields = ('name',) + + def clean_name(self, attrs, source): + """ prevent users creating subdomains of other users domains """ + name = attrs[source] + parent = Domain.objects.get_parent(name) + if parent and parent.account != self.account: + raise ValidationError(_("Can not create subdomains of other users domains")) + return attrs + + def validate(self, data): + """ Checks if everything is consistent """ + data = super(DomainSerializer, self).validate(data) + name = data.get('name') + if name: + instance = self.instance + if instance is None: + instance = Domain(name=name, account=self.account) + records = data['records'] + domain = domain_for_validation(instance, records) + validators.validate_zone(domain.render_zone()) + return data + + def create(self, validated_data): + records = validated_data.pop('records') + domain = super(DomainSerializer, self).create(validated_data) + for record in records: + domain.records.create(type=record['type'], value=record['value']) + return domain + + def update(self, instance, validated_data): + precords = validated_data.pop('records') + domain = super(DomainSerializer, self).update(instance, validated_data) + to_delete = [] + for erecord in domain.records.all(): + match = False + for ix, precord in enumerate(precords): + if erecord.type == precord['type'] and erecord.value == precord['value']: + match = True + break + if match: + precords.pop(ix) + else: + to_delete.append(erecord) + for precord in precords: + try: + recycled = to_delete.pop() + except IndexError: + domain.records.create(type=precord['type'], value=precord['value']) + else: + recycled.type = precord['type'] + recycled.value = precord['value'] + recycled.save() + return domain diff --git a/orchestra/contrib/domains/settings.py b/orchestra/contrib/domains/settings.py new file mode 100644 index 00000000..78b5fe90 --- /dev/null +++ b/orchestra/contrib/domains/settings.py @@ -0,0 +1,127 @@ +from orchestra.contrib.settings import Setting +from orchestra.core.validators import validate_ipv4_address, validate_ipv6_address, validate_ip_address +from orchestra.settings import ORCHESTRA_BASE_DOMAIN + +from .validators import validate_zone_interval, validate_mx_record, validate_domain_name + + +DOMAINS_DEFAULT_NAME_SERVER = Setting('DOMAINS_DEFAULT_NAME_SERVER', + 'ns.{}'.format(ORCHESTRA_BASE_DOMAIN), + validators=[validate_domain_name], + help_text="Uses ORCHESTRA_BASE_DOMAIN by default." +) + + +DOMAINS_DEFAULT_HOSTMASTER = Setting('DOMAINS_DEFAULT_HOSTMASTER', + 'hostmaster@{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default." +) + + +DOMAINS_DEFAULT_TTL = Setting('DOMAINS_DEFAULT_TTL', + '1h', + validators=[validate_zone_interval], +) + + +DOMAINS_DEFAULT_REFRESH = Setting('DOMAINS_DEFAULT_REFRESH', + '1d', + validators=[validate_zone_interval], +) + + +DOMAINS_DEFAULT_RETRY = Setting('DOMAINS_DEFAULT_RETRY', + '2h', + validators=[validate_zone_interval], +) + + +DOMAINS_DEFAULT_EXPIRE = Setting('DOMAINS_DEFAULT_EXPIRE', + '4w', + validators=[validate_zone_interval], +) + + +DOMAINS_DEFAULT_MIN_TTL = Setting('DOMAINS_DEFAULT_MIN_TTL', + '1h', + validators=[validate_zone_interval], +) + + +DOMAINS_ZONE_PATH = Setting('DOMAINS_ZONE_PATH', + '/etc/bind/master/%(name)s' +) + + +DOMAINS_MASTERS_PATH = Setting('DOMAINS_MASTERS_PATH', + '/etc/bind/named.conf.local', +) + + +DOMAINS_SLAVES_PATH = Setting('DOMAINS_SLAVES_PATH', + '/etc/bind/named.conf.local', +) + + +DOMAINS_CHECKZONE_BIN_PATH = Setting('DOMAINS_CHECKZONE_BIN_PATH', + 'named-checkzone -i local -k fail -n fail', +) + + +DOMAINS_ZONE_VALIDATION_TMP_DIR = Setting('DOMAINS_ZONE_VALIDATION_TMP_DIR', + '/dev/shm', + help_text="Used for creating temporary zone files used for validation." +) + + +DOMAINS_DEFAULT_A = Setting('DOMAINS_DEFAULT_A', + '10.0.3.13', + validators=[validate_ipv4_address] +) + + +DOMAINS_DEFAULT_AAAA = Setting('DOMAINS_DEFAULT_AAAA', '', + validators=[validate_ipv6_address] +) + + +DOMAINS_DEFAULT_MX = Setting('DOMAINS_DEFAULT_MX', + default=( + '10 mail.{}.'.format(ORCHESTRA_BASE_DOMAIN), + '10 mail2.{}.'.format(ORCHESTRA_BASE_DOMAIN), + ), + validators=[lambda mxs: list(map(validate_mx_record, mxs))], + help_text="Uses ORCHESTRA_BASE_DOMAIN by default." +) + + +DOMAINS_DEFAULT_NS = Setting('DOMAINS_DEFAULT_NS', + default=( + 'ns1.{}.'.format(ORCHESTRA_BASE_DOMAIN), + 'ns2.{}.'.format(ORCHESTRA_BASE_DOMAIN), + ), + validators=[lambda nss: list(map(validate_domain_name, nss))], + help_text="Uses ORCHESTRA_BASE_DOMAIN by default." +) + + +DOMAINS_FORBIDDEN = Setting('DOMAINS_FORBIDDEN', + '', + help_text=( + "This setting prevents users from providing random domain names, i.e. google.com
    " + "You can generate a 5K forbidden domains list from Alexa's top 1M:
    " + " wget http://s3.amazonaws.com/alexa-static/top-1m.csv.zip -O /tmp/top-1m.csv.zip && " + "unzip -p /tmp/top-1m.csv.zip | head -n 5000 | sed 's/^.*,//' > forbidden_domains.list
    " + "'%(site_dir)s/forbidden_domains.list')" + ) +) + + +DOMAINS_MASTERS = Setting('DOMAINS_MASTERS', + (), + validators=[lambda masters: list(map(validate_ip_address, masters))], + help_text="Additional master server ip addresses other than autodiscovered by router.get_servers()." +) + +#TODO remove pangea-specific default +DOMAINS_DEFAULT_DNS2136 = "key pangea.key;" diff --git a/orchestra/contrib/domains/templates/admin/domains/domain/change_form.html b/orchestra/contrib/domains/templates/admin/domains/domain/change_form.html new file mode 100644 index 00000000..5bf4ffb4 --- /dev/null +++ b/orchestra/contrib/domains/templates/admin/domains/domain/change_form.html @@ -0,0 +1,15 @@ +{% extends "admin/change_form.html" %} +{% load i18n admin_urls static admin_modify %} + + +{% block object-tools-items %} +
  • + {% trans "View zone" %} +
    • +
  • + {% url opts|admin_urlname:'history' original.pk|admin_urlquote as history_url %} + {% trans "History" %} +
    • +{% if has_absolute_url %}
  • {% trans "View on site" %}
    • {% endif%} +{% endblock %} + diff --git a/orchestra/contrib/domains/templates/admin/domains/domain/edit_records.html b/orchestra/contrib/domains/templates/admin/domains/domain/edit_records.html new file mode 100644 index 00000000..48d3a0bd --- /dev/null +++ b/orchestra/contrib/domains/templates/admin/domains/domain/edit_records.html @@ -0,0 +1,20 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} +{% load static %} + + +{% block extrahead %} +{{ block.super }} + + + + + + + +{% endblock %} + +{% block formset %} + {% for formset in formsets %} + {{ formset.as_admin }} + {% endfor %} +{% endblock %} diff --git a/orchestra/contrib/domains/templates/admin/domains/domain/view_zone.html b/orchestra/contrib/domains/templates/admin/domains/domain/view_zone.html new file mode 100644 index 00000000..838e1073 --- /dev/null +++ b/orchestra/contrib/domains/templates/admin/domains/domain/view_zone.html @@ -0,0 +1,22 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation{% endblock %} + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› {{ opts.app_label|capfirst }} +› {{ opts.verbose_name_plural|capfirst|escape }} +› {{ object|truncatewords:"18" }} +› {% trans 'View zone' %} +
    +{% endblock %} + +{% block content %} + +
    +{{ object.render_zone }}
+
    +{% endblock %} + diff --git a/orchestra/contrib/domains/tests/__init__.py b/orchestra/contrib/domains/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/domains/tests/functional_tests/__init__.py b/orchestra/contrib/domains/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/domains/tests/functional_tests/tests.py b/orchestra/contrib/domains/tests/functional_tests/tests.py new file mode 100644 index 00000000..f13342e4 --- /dev/null +++ b/orchestra/contrib/domains/tests/functional_tests/tests.py @@ -0,0 +1,325 @@ +import os +import time +import socket +from functools import partial + +from django.conf import settings as djsettings +from django.urls import reverse +from selenium.webdriver.support.select import Select + +from orchestra.contrib.orchestration.models import Server, Route +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error, save_response_on_error +from orchestra.utils.sys import run + +from ... import settings, utils, backends +from ...models import Domain, Record + + +run = partial(run, display=False) + + +class DomainTestMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_MASTER_SERVER', 'localhost') + SLAVE_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') + MASTER_SERVER_ADDR = socket.gethostbyname(MASTER_SERVER) + SLAVE_SERVER_ADDR = socket.gethostbyname(SLAVE_SERVER) + + def setUp(self): + djsettings.DEBUG = True + super(DomainTestMixin, self).setUp() + self.domain_name = 'orchestra%s.lan' % random_ascii(10) + self.domain_records = ( + (Record.MX, '10 mail.orchestra.lan.'), + (Record.MX, '20 mail2.orchestra.lan.'), + (Record.NS, 'ns1.%s.' % self.domain_name), + (Record.NS, 'ns2.%s.' % self.domain_name), + ) + self.domain_update_records = ( + (Record.MX, '30 mail3.orchestra.lan.'), + (Record.MX, '40 mail4.orchestra.lan.'), + (Record.NS, 'ns1.%s.' % self.domain_name), + (Record.NS, 'ns2.%s.' % self.domain_name), + ) + self.ns1_name = 'ns1.%s' % self.domain_name + self.ns1_records = ( + (Record.A, self.SLAVE_SERVER_ADDR), + ) + self.ns2_name = 'ns2.%s' % self.domain_name + self.ns2_records = ( + (Record.A, self.MASTER_SERVER_ADDR), + ) + self.www_name = 'www.%s' % self.domain_name + self.www_records = ( + (Record.CNAME, 'external.server.org.'), + ) + self.django_domain_name = 'django%s.lan' % random_ascii(10) + + def add_route(self): + raise NotImplementedError + + def add(self, domain_name, records): + raise NotImplementedError + + def delete(self, domain_name, records): + raise NotImplementedError + + def update(self, domain_name, records): + raise NotImplementedError + + def validate_add(self, server_addr, domain_name): + context = { + 'domain_name': domain_name, + 'server_addr': server_addr + } + dig_soa = 'dig @%(server_addr)s %(domain_name)s SOA|grep "\sSOA\s"' + soa = run(dig_soa % context).stdout.split() + # testdomain.org. 3600 IN SOA ns.example.com. hostmaster.example.com. 2014021100 86400 7200 2419200 3600 + self.assertEqual('%(domain_name)s.' % context, soa[0]) + self.assertEqual('3600', soa[1]) + self.assertEqual('IN', soa[2]) + self.assertEqual('SOA', soa[3]) + self.assertEqual('%s.' % settings.DOMAINS_DEFAULT_NAME_SERVER, soa[4]) + hostmaster = utils.format_hostmaster(settings.DOMAINS_DEFAULT_HOSTMASTER) + self.assertEqual(hostmaster, soa[5]) + + dig_ns = 'dig @%(server_addr)s %(domain_name)s NS|grep "\sNS\s"' + name_servers = run(dig_ns % context).stdout + # testdomain.org. 3600 IN NS ns1.orchestra.lan. + ns_records = ['ns1.%s.' % self.domain_name, 'ns2.%s.' % self.domain_name] + self.assertEqual(2, len(name_servers.splitlines())) + for ns in name_servers.splitlines(): + ns = ns.split() + # testdomain.org. 3600 IN NS ns1.orchestra.lan. + self.assertEqual('%(domain_name)s.' % context, ns[0]) + self.assertEqual('3600', ns[1]) + self.assertEqual('IN', ns[2]) + self.assertEqual('NS', ns[3]) + self.assertIn(ns[4], ns_records) + + dig_mx = 'dig @%(server_addr)s %(domain_name)s MX|grep "\sMX\s"' + mail_servers = run(dig_mx % context).stdout + for mx in mail_servers.splitlines(): + mx = mx.split() + # testdomain.org. 3600 IN NS ns1.orchestra.lan. + self.assertEqual('%(domain_name)s.' % context, mx[0]) + self.assertEqual('3600', mx[1]) + self.assertEqual('IN', mx[2]) + self.assertEqual('MX', mx[3]) + self.assertIn(mx[4], ['10', '20']) + self.assertIn(mx[5], ['mail2.orchestra.lan.', 'mail.orchestra.lan.']) + + def validate_delete(self, server_addr, domain_name): + context = { + 'domain_name': domain_name, + 'server_addr': server_addr + } + dig_soa = 'dig @%(server_addr)s %(domain_name)s|grep "\sSOA\s"' + soa = run(dig_soa % context, valid_codes=(0, 1)).stdout + if soa: + soa = soa.split() + self.assertEqual('IN', soa[2]) + self.assertEqual('SOA', soa[3]) + self.assertNotEqual('%s.' % settings.DOMAINS_DEFAULT_NAME_SERVER, soa[4]) + hostmaster = utils.format_hostmaster(settings.DOMAINS_DEFAULT_HOSTMASTER) + self.assertNotEqual(hostmaster, soa[5]) + + def validate_update(self, server_addr, domain_name): + context = { + 'domain_name': domain_name, + 'server_addr': server_addr + } + dig_soa = 'dig @%(server_addr)s %(domain_name)s SOA | grep "\sSOA\s"' + soa = run(dig_soa % context).stdout.split() + # testdomain.org. 3600 IN SOA ns.example.com. hostmaster.example.com. 2014021100 86400 7200 2419200 3600 + self.assertEqual('%(domain_name)s.' % context, soa[0]) + self.assertEqual('3600', soa[1]) + self.assertEqual('IN', soa[2]) + self.assertEqual('SOA', soa[3]) + self.assertEqual('%s.' % settings.DOMAINS_DEFAULT_NAME_SERVER, soa[4]) + hostmaster = utils.format_hostmaster(settings.DOMAINS_DEFAULT_HOSTMASTER) + self.assertEqual(hostmaster, soa[5]) + + dig_ns = 'dig @%(server_addr)s %(domain_name)s NS |grep "\sNS\s"' + name_servers = run(dig_ns % context).stdout + ns_records = ['ns1.%s.' % self.domain_name, 'ns2.%s.' % self.domain_name] + self.assertEqual(2, len(name_servers.splitlines())) + for ns in name_servers.splitlines(): + ns = ns.split() + # testdomain.org. 3600 IN NS ns1.orchestra.lan. + self.assertEqual('%(domain_name)s.' % context, ns[0]) + self.assertEqual('3600', ns[1]) + self.assertEqual('IN', ns[2]) + self.assertEqual('NS', ns[3]) + self.assertIn(ns[4], ns_records) + + dig_mx = 'dig @%(server_addr)s %(domain_name)s MX | grep "\sMX\s"' + mx = run(dig_mx % context).stdout.split() + # testdomain.org. 3600 IN MX 10 orchestra.lan. + self.assertEqual('%(domain_name)s.' % context, mx[0]) + self.assertEqual('3600', mx[1]) + self.assertEqual('IN', mx[2]) + self.assertEqual('MX', mx[3]) + self.assertIn(mx[4], ['30', '40']) + self.assertIn(mx[5], ['mail3.orchestra.lan.', 'mail4.orchestra.lan.']) + + def validate_www_update(self, server_addr, domain_name): + context = { + 'domain_name': domain_name, + 'server_addr': server_addr + } + dig_cname = 'dig @%(server_addr)s www.%(domain_name)s CNAME | grep "\sCNAME\s"' + cname = run(dig_cname % context).stdout.split() + # testdomain.org. 3600 IN MX 10 orchestra.lan. + self.assertEqual('www.%(domain_name)s.' % context, cname[0]) + self.assertEqual('3600', cname[1]) + self.assertEqual('IN', cname[2]) + self.assertEqual('CNAME', cname[3]) + self.assertEqual('external.server.org.', cname[4]) + + def test_add(self): + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) + self.add(self.domain_name, self.domain_records) +# self.addCleanup(self.delete, self.domain_name) + self.validate_add(self.MASTER_SERVER_ADDR, self.domain_name) + time.sleep(1) + self.validate_add(self.SLAVE_SERVER_ADDR, self.domain_name) + + def test_delete(self): + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) + self.add(self.domain_name, self.domain_records) + self.delete(self.domain_name) + for name in [self.domain_name, self.ns1_name, self.ns2_name]: + self.validate_delete(self.MASTER_SERVER_ADDR, name) + self.validate_delete(self.SLAVE_SERVER_ADDR, name) + + def test_update(self): + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) + self.add(self.domain_name, self.domain_records) + self.addCleanup(self.delete, self.domain_name) + self.update(self.domain_name, self.domain_update_records) + time.sleep(0.5) + self.validate_update(self.MASTER_SERVER_ADDR, self.domain_name) + time.sleep(5) + self.validate_update(self.SLAVE_SERVER_ADDR, self.domain_name) + self.add(self.www_name, self.www_records) + time.sleep(0.5) + self.validate_www_update(self.MASTER_SERVER_ADDR, self.domain_name) + time.sleep(5) + self.validate_www_update(self.SLAVE_SERVER_ADDR, self.domain_name) + + def test_add_add_delete_delete(self): + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) + self.add(self.domain_name, self.domain_records) + self.add(self.django_domain_name, self.domain_records) + self.delete(self.domain_name) + self.validate_add(self.MASTER_SERVER_ADDR, self.django_domain_name) + self.validate_add(self.SLAVE_SERVER_ADDR, self.django_domain_name) + self.delete(self.django_domain_name) + self.validate_delete(self.MASTER_SERVER_ADDR, self.django_domain_name) + self.validate_delete(self.SLAVE_SERVER_ADDR, self.django_domain_name) + + def test_bad_creation(self): + self.assertRaises((self.rest.ResponseStatusError, AssertionError), + self.add, self.domain_name, self.domain_records) + + +class AdminDomainMixin(DomainTestMixin): + def setUp(self): + super(AdminDomainMixin, self).setUp() + self.add_route() + self.admin_login() + + def _add_records(self, records): + self.selenium.find_element_by_link_text('Add another Record').click() + for i, record in zip(range(0, len(records)), records): + type, value = record + type_input = self.selenium.find_element_by_id('id_records-%d-type' % i) + type_select = Select(type_input) + type_select.select_by_value(type) + value_input = self.selenium.find_element_by_id('id_records-%d-value' % i) + value_input.clear() + value_input.send_keys(value) + return value_input + + @snapshot_on_error + def add(self, domain_name, records): + add = reverse('admin:domains_domain_add') + url = self.live_server_url + add + self.selenium.get(url) + + name = self.selenium.find_element_by_id('id_name') + name.send_keys(domain_name) + + account_input = self.selenium.find_element_by_id('id_account') + account_select = Select(account_input) + account_select.select_by_value(str(self.account.pk)) + + value_input = self._add_records(records) + value_input.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def delete(self, domain_name): + domain = Domain.objects.get(name=domain_name) + self.admin_delete(domain) + + @snapshot_on_error + def update(self, domain_name, records): + domain = Domain.objects.get(name=domain_name) + change = reverse('admin:domains_domain_change', args=(domain.pk,)) + url = self.live_server_url + change + self.selenium.get(url) + value_input = self._add_records(records) + value_input.submit() + self.assertNotEqual(url, self.selenium.current_url) + + +class RESTDomainMixin(DomainTestMixin): + def setUp(self): + super(RESTDomainMixin, self).setUp() + self.rest_login() + self.add_route() + + @save_response_on_error + def add(self, domain_name, records): + records = [ dict(type=type, value=value) for type,value in records ] + self.rest.domains.create(name=domain_name, records=records) + + @save_response_on_error + def delete(self, domain_name): + domain = Domain.objects.get(name=domain_name) + domain = self.rest.domains.retrieve(id=domain.pk) + domain.delete() + + @save_response_on_error + def update(self, domain_name, records): + records = [ dict(type=type, value=value) for type,value in records ] + domains = self.rest.domains.retrieve(name=domain_name) + domain = domains.get() + domain.update(records=records) + + +class Bind9BackendMixin(object): + DEPENDENCIES = ( + 'orchestra.contrib.orchestration', + ) + + def add_route(self): + master = Server.objects.create(name=self.MASTER_SERVER, address=self.MASTER_SERVER_ADDR) + backend = backends.Bind9MasterDomainController.get_name() + Route.objects.create(backend=backend, match=True, host=master) + slave = Server.objects.create(name=self.SLAVE_SERVER, address=self.SLAVE_SERVER_ADDR) + backend = backends.Bind9SlaveDomainController.get_name() + Route.objects.create(backend=backend, match=True, host=slave) + + +class RESTBind9BackendDomainTest(Bind9BackendMixin, RESTDomainMixin, BaseLiveServerTestCase): + pass + + +class AdminBind9BackendDomainTest(Bind9BackendMixin, AdminDomainMixin, BaseLiveServerTestCase): + pass diff --git a/orchestra/contrib/domains/tests/test_domains.py b/orchestra/contrib/domains/tests/test_domains.py new file mode 100644 index 00000000..f15ac2d3 --- /dev/null +++ b/orchestra/contrib/domains/tests/test_domains.py @@ -0,0 +1,18 @@ +from orchestra.utils.tests import BaseTestCase + +from ..models import Domain + + +class DomainTest(BaseTestCase): + def test_top_relation(self): + account = self.create_account() + domain = Domain.objects.create(name='rostrepalid.org', account=account) + Domain.objects.create(name='www.rostrepalid.org') + Domain.objects.create(name='mail.rostrepalid.org') + self.assertEqual(2, len(domain.subdomains.all())) + + def test_render_zone(self): + account = self.create_account() + domain = Domain.objects.create(name='rostrepalid.org', account=account) + domain.render_zone() + diff --git a/orchestra/contrib/domains/utils.py b/orchestra/contrib/domains/utils.py new file mode 100644 index 00000000..e644729f --- /dev/null +++ b/orchestra/contrib/domains/utils.py @@ -0,0 +1,52 @@ +from collections import defaultdict + +from django.utils import timezone + + +class RecordStorage(object): + """ + list-dict implementation for fast lookups of record types + """ + + def __init__(self, *args): + self.records = list(*args) + self.type = defaultdict(list) + + def __iter__(self): + return iter(self.records) + + def append(self, record): + self.records.append(record) + self.type[record['type']].append(record) + + def insert(self, ix, record): + self.records.insert(ix, record) + self.type[record['type']].insert(ix, record) + + def by_type(self, type): + return self.type[type] + + +def generate_zone_serial(): + today = timezone.now() + return int("%.4d%.2d%.2d%.2d" % (today.year, today.month, today.day, 0)) + + +def format_hostmaster(hostmaster): + """ + The DNS encodes the as a single label, and encodes the + as a domain name. The single label from the + is prefaced to the domain name from to form the domain + name corresponding to the mailbox. Thus the mailbox HOSTMASTER@SRI- + NIC.ARPA is mapped into the domain name HOSTMASTER.SRI-NIC.ARPA. If the + contains dots or other special characters, its + representation in a master file will require the use of backslash + quoting to ensure that the domain name is properly encoded. For + example, the mailbox Action.domains@ISI.EDU would be represented as + Action\.domains.ISI.EDU. + http://www.ietf.org/rfc/rfc1035.txt + """ + name, domain = hostmaster.split('@') + if '.' in name: + name = name.replace('.', '\.') + return "%s.%s." % (name, domain) diff --git a/orchestra/contrib/domains/validators.py b/orchestra/contrib/domains/validators.py new file mode 100644 index 00000000..4722493a --- /dev/null +++ b/orchestra/contrib/domains/validators.py @@ -0,0 +1,137 @@ +import logging +import os +import re + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra.core.validators import validate_hostname +from orchestra.utils import paths +from orchestra.utils.sys import run + +from .. import domains + + +logger = logging.getLogger(__name__) + + +def validate_allowed_domain(value): + context = { + 'site_dir': paths.get_site_dir() + } + fname = domains.settings.DOMAINS_FORBIDDEN + if fname: + fname = fname % context + with open(fname, 'r') as forbidden: + for domain in forbidden.readlines(): + if re.match(r'^(.*\.)*%s$' % domain.strip(), value): + raise ValidationError(_("This domain name is not allowed")) + + +def validate_domain_name(value): + # SRV, CNAME and TXT records may use '_' in the domain name + value = value.lstrip('*.').replace('_', '') + try: + validate_hostname(value) + except ValidationError: + raise ValidationError(_("Not a valid domain name.")) + + +def validate_zone_interval(value): + try: + int(value) + except ValueError: + value, magnitude = value[:-1], value[-1] + if magnitude not in ('s', 'm', 'h', 'd', 'w') or not value.isdigit(): + msg = _("%s is not an appropiate zone interval value") % value + raise ValidationError(msg) + + +def validate_zone_label(value): + """ + Allowable characters in a label for a host name are only ASCII letters, digits, and the `-' character. + Labels may not be all numbers, but may have a leading digit (e.g., 3com.com). + Labels must end and begin only with a letter or digit. See [RFC 1035] and [RFC 1123]. + """ + if not re.match(r'^[a-z0-9][\.\-0-9a-z]*[\.0-9a-z]$', value): + msg = _("Labels must start and end with a letter or digit, " + "and have as interior characters only letters, digits, and hyphen.") + raise ValidationError(msg) + if not value.endswith('.'): + msg = _("Use a fully expanded domain name ending with a dot.") + raise ValidationError(msg) + if len(value) > 254: + raise ValidationError(_("Labels must be 63 characters or less.")) + + +def validate_mx_record(value): + msg = _("MX record format is 'priority domain.' tuple, with priority being a number.") + value = value.split() + if len(value) != 2: + raise ValidationError(msg) + else: + try: + int(value[0]) + except ValueError: + raise ValidationError(msg) + value = value[1] + validate_zone_label(value) + + +def validate_srv_record(value): + # 1 0 9 server.example.com. + msg = _("%s is not an appropiate SRV record value") % value + value = value.split() + for i in [0,1,2]: + try: + int(value[i]) + except ValueError: + raise ValidationError(msg) + validate_zone_label(value[-1]) + + +def validate_soa_record(value): + # ns1.pangea.ORG. hostmaster.pangea.ORG. 2012010401 28800 7200 604800 86400 + msg = _("%s is not an appropiate SRV record value") % value + values = value.split() + if len(values) != 7: + raise ValidationError(msg) + validate_zone_label(values[0]) + validate_zone_label(values[1]) + for value in values[2:]: + try: + int(value) + except ValueError: + raise ValidationError(msg) + + +def validate_quoted_record(value): + value = value.strip() + if ' ' in value and (value[0] != '"' or value[-1] != '"'): + raise ValidationError( + _("This record value contains spaces, you must enclose the string in double quotes; " + "otherwise, individual words will be separately quoted and break up the record " + "into multiple parts.") + ) + + +def validate_zone(zone): + """ Ultimate zone file validation using named-checkzone """ + zone_name = zone.split()[0][:-1] + zone_path = os.path.join(domains.settings.DOMAINS_ZONE_VALIDATION_TMP_DIR, zone_name) + checkzone = domains.settings.DOMAINS_CHECKZONE_BIN_PATH + try: + with open(zone_path, 'wb') as f: + f.write(zone.encode('ascii')) + # Don't use /dev/stdin becuase the 'argument list is too long' error + check = run(' '.join([checkzone, zone_name, zone_path]), valid_codes=(0,1,127), display=False) + finally: + try: + os.unlink(zone_path) + except FileNotFoundError: + pass + if check.exit_code == 127: + logger.error("Cannot validate domain zone: %s not installed." % checkzone) + elif check.exit_code == 1: + errors = re.compile(r'zone.*: (.*)').findall(check.stdout.decode('utf8'))[:-1] + raise ValidationError(', '.join(errors)) diff --git a/orchestra/contrib/history/__init__.py b/orchestra/contrib/history/__init__.py new file mode 100644 index 00000000..d39042d6 --- /dev/null +++ b/orchestra/contrib/history/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.history.apps.HistoryConfig' diff --git a/orchestra/contrib/history/admin.py b/orchestra/contrib/history/admin.py new file mode 100644 index 00000000..903eeda0 --- /dev/null +++ b/orchestra/contrib/history/admin.py @@ -0,0 +1,130 @@ +from django.contrib import admin +from django.templatetags.static import static +from django.contrib.admin.templatetags.admin_urls import add_preserved_filters +from django.contrib.admin.utils import unquote +from django.http import HttpResponseRedirect +from django.urls import NoReverseMatch, reverse +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin.utils import admin_date, admin_link + + +class LogEntryAdmin(admin.ModelAdmin): + list_display = ( + 'display_action_time', 'user_link', 'display_message', + ) + list_filter = ( + 'action_flag', + ('user', admin.RelatedOnlyFieldListFilter), + ('content_type', admin.RelatedOnlyFieldListFilter), + ) + date_hierarchy = 'action_time' + search_fields = ('object_repr', 'change_message', 'user__username') + fields = ( + 'user_link', 'content_object_link', 'display_action_time', 'display_action', + 'change_message' + ) + readonly_fields = ( + 'user_link', 'content_object_link', 'display_action_time', 'display_action', + ) + actions = None + list_select_related = ('user', 'content_type') + list_display_links = None + + user_link = admin_link('user') + display_action_time = admin_date('action_time', short_description=_("Time")) + + @mark_safe + def display_message(self, log): + edit = format_html('', **{ + 'url': reverse('admin:admin_logentry_change', args=(log.pk,)), + 'img': static('admin/img/icon-changelink.svg'), + }) + if log.is_addition(): + return _('Added "%(link)s". %(edit)s') % { + 'link': self.content_object_link(log), + 'edit': edit + } + elif log.is_change(): + return _('Changed "%(link)s" - %(changes)s %(edit)s') % { + 'link': self.content_object_link(log), + 'changes': log.get_change_message(), + 'edit': edit, + } + elif log.is_deletion(): + return _('Deleted "%(object)s." %(edit)s') % { + 'object': log.object_repr, + 'edit': edit, + } + display_message.short_description = _("Message") + display_message.admin_order_field = 'action_flag' + + def display_action(self, log): + if log.is_addition(): + return _("Added") + elif log.is_change(): + return _("Changed") + return _("Deleted") + display_action.short_description = _("Action") + display_action.admin_order_field = 'action_flag' + + def content_object_link(self, log): + ct = log.content_type + view = 'admin:%s_%s_change' % (ct.app_label, ct.model) + try: + url = reverse(view, args=(log.object_id,)) + except NoReverseMatch: + return log.object_repr + return format_html('{}', url, log.object_repr) + content_object_link.short_description = _("Content object") + content_object_link.admin_order_field = 'object_repr' + + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): + """ Add rel_opts and object to context """ + if not add and 'edit' in request.GET.urlencode(): + context.update({ + 'rel_opts': obj.content_type.model_class()._meta, + 'object': obj, + }) + return super(LogEntryAdmin, self).render_change_form( + request, context, add, change, form_url, obj) + + def response_change(self, request, obj): + """ save and continue preserve edit query string """ + response = super(LogEntryAdmin, self).response_change(request, obj) + if 'edit' in request.GET.urlencode() and 'edit' not in response.url: + return HttpResponseRedirect(response.url + '?edit=True') + return response + + def response_post_save_change(self, request, obj): + """ save redirect to object history """ + if 'edit' in request.GET.urlencode(): + opts = obj.content_type.model_class()._meta + view = 'admin:%s_%s_history' % (opts.app_label, opts.model_name) + post_url = reverse(view, args=(obj.object_id,)) + preserved_filters = self.get_preserved_filters(request) + post_url = add_preserved_filters({ + 'preserved_filters': preserved_filters, 'opts': opts + }, post_url) + return HttpResponseRedirect(post_url) + return super(LogEntryAdmin, self).response_post_save_change(request, obj) + + def has_add_permission(self, *args, **kwargs): + return False + + def has_delete_permission(self, *args, **kwargs): + return False + + def log_addition(self, *args, **kwargs): + pass + + def log_change(self, *args, **kwargs): + pass + + def log_deletion(self, *args, **kwargs): + pass + + +admin.site.register(admin.models.LogEntry, LogEntryAdmin) diff --git a/orchestra/contrib/history/apps.py b/orchestra/contrib/history/apps.py new file mode 100644 index 00000000..8d168e5c --- /dev/null +++ b/orchestra/contrib/history/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + +from orchestra.core import administration + + +class HistoryConfig(AppConfig): + name = 'orchestra.contrib.history' + verbose_name = 'History' + + def ready(self): + from django.contrib.admin.models import LogEntry + administration.register( + LogEntry, verbose_name='History', verbose_name_plural='History', icon='History.png' + ) diff --git a/orchestra/contrib/history/templates/admin/admin/logentry/change_form.html b/orchestra/contrib/history/templates/admin/admin/logentry/change_form.html new file mode 100644 index 00000000..95c693c0 --- /dev/null +++ b/orchestra/contrib/history/templates/admin/admin/logentry/change_form.html @@ -0,0 +1,22 @@ +{% extends "admin/change_form.html" %} +{% load i18n admin_urls %} + +{% block object-tools-items %} +{% endblock %} + + +{% block breadcrumbs %} +{% if rel_opts %} +
    +{% trans 'Home' %} +› {{ rel_opts.app_config.verbose_name }} +› {{ rel_opts.model_name|capfirst }} +› {{ object.object_repr|truncatewords:"18" }} +› {% trans 'History' %} +› {% trans 'Edit' %} +
    +{% else %} +{{ block.super }} +{% endif %} +{% endblock %} + diff --git a/orchestra/contrib/history/templates/admin/object_history.html b/orchestra/contrib/history/templates/admin/object_history.html new file mode 100644 index 00000000..9441b522 --- /dev/null +++ b/orchestra/contrib/history/templates/admin/object_history.html @@ -0,0 +1,43 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls static %} + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ module_name }} +› {{ object|truncatewords:"18" }} +› {% trans 'History' %} +
    +{% endblock %} + +{% block content %} +
    +
    + +{% if action_list %} + + + + + + + + + + {% for action in action_list %} + + + + + + {% endfor %} + +
    {% trans 'Date/time' %}{% trans 'User' %}{% trans 'Action' %}
    {{ action.action_time|date:"DATETIME_FORMAT" }}{{ action.user.get_username }}{% if action.user.get_full_name %} ({{ action.user.get_full_name }}){% endif %}{% if action.is_addition and not action.change_message %}{% trans 'Added' %}{% else %}{{ action.change_message }}{% endif %}
    +{% else %} +

    {% trans "This object doesn't have a change history. It probably wasn't added via this admin site." %}

    +{% endif %} +
    +
    +{% endblock %} + diff --git a/orchestra/contrib/issues/__init__.py b/orchestra/contrib/issues/__init__.py new file mode 100644 index 00000000..650ba7fe --- /dev/null +++ b/orchestra/contrib/issues/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.issues.apps.IssuesConfig' diff --git a/orchestra/contrib/issues/actions.py b/orchestra/contrib/issues/actions.py new file mode 100644 index 00000000..dac29765 --- /dev/null +++ b/orchestra/contrib/issues/actions.py @@ -0,0 +1,137 @@ +import sys + +from django.contrib import messages +from django.db import transaction +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.admin.decorators import action_with_confirmation + +from .forms import ChangeReasonForm +from .helpers import markdown_formated_changes +from .models import Queue, Ticket + + +def change_ticket_state_factory(action, verbose_name, final_state): + context = { + 'action': action, + 'form': ChangeReasonForm() + } + @transaction.atomic + @action_with_confirmation(action_name=action, extra_context=context) + def change_ticket_state(modeladmin, request, queryset, action=action, final_state=final_state): + form = ChangeReasonForm(request.POST) + if form.is_valid(): + reason = form.cleaned_data['reason'] + for ticket in queryset: + if ticket.state != final_state: + changes = { + 'state': (ticket.state, final_state) + } + is_read = ticket.is_read_by(request.user) + getattr(ticket, action)() + msg = _("Marked as %s") % final_state.lower() + modeladmin.log_change(request, ticket, msg) + content = markdown_formated_changes(changes) + content += reason + ticket.messages.create(content=content, author=request.user) + if is_read and not ticket.is_read_by(request.user): + ticket.mark_as_read_by(request.user) + context = { + 'count': queryset.count(), + 'state': final_state.lower() + } + msg = _("%(count)s selected tickets are now %(state)s.") % context + modeladmin.message_user(request, msg) + else: + context['form'] = form + # action_with_confirmation must display form validation errors + return True + change_ticket_state.url_name = action + change_ticket_state.tool_description = verbose_name + change_ticket_state.short_description = _('%s selected tickets') % verbose_name + change_ticket_state.help_text = _('Mark ticket as %s.') % final_state.lower() + change_ticket_state.__name__ = action + return change_ticket_state + + +action_map = { + Ticket.RESOLVED: ('resolve', _("Resolve")), + Ticket.REJECTED: ('reject', _("Reject")), + Ticket.CLOSED: ('close', _("Close")), +} + + +thismodule = sys.modules[__name__] +for state, names in action_map.items(): + name, verbose_name = names + action = change_ticket_state_factory(name, verbose_name, state) + setattr(thismodule, '%s_tickets' % name, action) + + +@transaction.atomic +def take_tickets(modeladmin, request, queryset): + for ticket in queryset: + if ticket.owner != request.user: + changes = { + 'owner': (ticket.owner, request.user) + } + is_read = ticket.is_read_by(request.user) + ticket.take(request.user) + modeladmin.log_change(request, ticket, _("Taken")) + content = markdown_formated_changes(changes) + ticket.messages.create(content=content, author=request.user) + if is_read and not ticket.is_read_by(request.user): + ticket.mark_as_read_by(request.user) + modeladmin.log_change(request, ticket, 'Taken') + context = { + 'count': queryset.count(), + 'user': request.user + } + msg = _("%(count)s selected tickets are now owned by %(user)s.") % context + modeladmin.message_user(request, msg) +take_tickets.url_name = 'take' +take_tickets.tool_description = _("Take") +take_tickets.short_description = _("Take selected tickets") +take_tickets.help_text = _("Make yourself owner of the ticket.") + + +@transaction.atomic +def mark_as_unread(modeladmin, request, queryset): + """ Mark a tickets as unread """ + for ticket in queryset: + ticket.mark_as_unread_by(request.user) + modeladmin.log_change(request, ticket, 'Marked as unread') + num = len(queryset) + msg = ngettext( + _("Selected ticket has been marked as unread."), + _("%i selected tickets have been marked as unread.") % num, + num) + modeladmin.message_user(request, msg) + + +@transaction.atomic +def mark_as_read(modeladmin, request, queryset): + """ Mark a tickets as unread """ + for ticket in queryset: + ticket.mark_as_read_by(request.user) + modeladmin.log_change(request, ticket, 'Marked as read') + num = len(queryset) + msg = ngettext( + _("Selected ticket has been marked as read."), + _("%i selected tickets have been marked as read.") % num, + num) + modeladmin.message_user(request, msg) + + +@transaction.atomic +def set_default_queue(modeladmin, request, queryset): + """ Set a queue as default issues queue """ + if queryset.count() != 1: + messages.warning(request, _("Please, select only one queue.")) + return + Queue.objects.filter(default=True).update(default=False) + queue = queryset.get() + queue.default = True + queue.save(update_fields=['default']) + modeladmin.log_change(request, queue, _("Chosen as default.")) + messages.info(request, _("Chosen '%s' as default queue.") % queue) diff --git a/orchestra/contrib/issues/admin.py b/orchestra/contrib/issues/admin.py new file mode 100644 index 00000000..25ae3f21 --- /dev/null +++ b/orchestra/contrib/issues/admin.py @@ -0,0 +1,323 @@ +from django import forms +from django.urls import re_path as url +from django.contrib import admin +from django.urls import reverse +from django.db import models +from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from django.utils.html import format_html, strip_tags +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ +from markdown import markdown + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import admin_link, admin_colored, wrap_admin_view, admin_date +from orchestra.contrib.contacts.models import Contact + +from .actions import (reject_tickets, resolve_tickets, take_tickets, close_tickets, + mark_as_unread, mark_as_read, set_default_queue) +from .filters import MyTicketsListFilter, TicketStateListFilter +from .forms import MessageInlineForm, TicketForm +from .helpers import get_ticket_changes, markdown_formated_changes, filter_actions +from .models import Ticket, Queue, Message + + +PRIORITY_COLORS = { + Ticket.HIGH: 'red', + Ticket.MEDIUM: 'darkorange', + Ticket.LOW: 'green', +} + + +STATE_COLORS = { + Ticket.NEW: 'grey', + Ticket.IN_PROGRESS: 'darkorange', + Ticket.FEEDBACK: 'purple', + Ticket.RESOLVED: 'green', + Ticket.REJECTED: 'firebrick', + Ticket.CLOSED: 'grey', +} + + +class MessageReadOnlyInline(admin.TabularInline): + model = Message + extra = 0 + can_delete = False + fields = ('content_html',) + readonly_fields = ('content_html',) + + class Media: + css = { + 'all': ('orchestra/css/hide-inline-id.css',) + } + + @mark_safe + def content_html(self, msg): + context = { + 'number': msg.number, + 'time': admin_date('created_at')(msg), + 'author': admin_link('author')(msg) if msg.author else msg.author_name, + } + summary = _("#%(number)i Updated by %(author)s about %(time)s") % context + header = '%s
    ' % summary + + content = markdown(msg.content) + content = content.replace('>\n', '>') + content = '
    %s
    ' % content + + return header + content + content_html.short_description = _("Content") + + def has_add_permission(self, request, obj): + return False + + def has_delete_permission(self, request, obj=None): + return False + + +class MessageInline(admin.TabularInline): + model = Message + extra = 1 + max_num = 1 + form = MessageInlineForm + can_delete = False + fields = ('content',) + + def get_formset(self, request, obj=None, **kwargs): + """ hook request.user on the inline form """ + self.form.user = request.user + return super(MessageInline, self).get_formset(request, obj, **kwargs) + + def get_queryset(self, request): + """ Don't show any message """ + qs = super(MessageInline, self).get_queryset(request) + return qs.none() + + +class TicketInline(admin.TabularInline): + fields = ( + 'ticket_id', 'subject', 'creator_link', 'owner_link', 'colored_state', + 'colored_priority', 'created', 'updated' + ) + readonly_fields = ( + 'ticket_id', 'subject', 'creator_link', 'owner_link', 'colored_state', + 'colored_priority', 'created', 'updated' + ) + model = Ticket + extra = 0 + max_num = 0 + + creator_link = admin_link('creator') + owner_link = admin_link('owner') + created = admin_link('created_at') + updated = admin_link('updated_at') + colored_state = admin_colored('state', colors=STATE_COLORS, bold=False) + colored_priority = admin_colored('priority', colors=PRIORITY_COLORS, bold=False) + + @mark_safe + def ticket_id(self, instance): + return '%s' % admin_link()(instance) + ticket_id.short_description = '#' + + +class TicketAdmin(ExtendedModelAdmin): + list_display = ( + 'unbold_id', 'bold_subject', 'display_creator', 'display_owner', + 'display_queue', 'display_priority', 'display_state', 'updated' + ) + list_display_links = ('unbold_id', 'bold_subject') + list_filter = ( + MyTicketsListFilter, 'queue', 'priority', TicketStateListFilter, + ) + default_changelist_filters = ( + ('state', 'OPEN'), + ) + date_hierarchy = 'created_at' + search_fields = ( + 'id', 'subject', 'creator__username', 'creator__email', 'queue__name', + 'owner__username' + ) + actions = ( + mark_as_unread, mark_as_read, reject_tickets, + resolve_tickets, close_tickets, take_tickets + ) + sudo_actions = ('delete_selected',) + change_view_actions = ( + resolve_tickets, close_tickets, reject_tickets, take_tickets + ) +# change_form_template = "admin/orchestra/change_form.html" + form = TicketForm + add_inlines = () + inlines = (MessageReadOnlyInline, MessageInline) + readonly_fields = ( + 'display_summary', 'display_queue', 'display_owner', 'display_state', + 'display_priority' + ) + readonly_fieldsets = ( + (None, { + 'fields': ('display_summary', + ('display_queue', 'display_owner'), + ('display_state', 'display_priority'), + 'display_description') + }), + ) + fieldsets = readonly_fieldsets + ( + ('Update', { + 'classes': ('collapse',), + 'fields': ('subject', + ('queue', 'owner',), + ('state', 'priority'), + 'description') + }), + ) + add_fieldsets = ( + (None, { + 'fields': ('subject', + ('queue', 'owner',), + ('state', 'priority'), + 'description') + }), + ) + list_select_related = ('queue', 'owner', 'creator') + + class Media: + css = { + 'all': ('issues/css/ticket-admin.css',) + } + js = ( + 'issues/js/ticket-admin.js', + ) + + display_creator = admin_link('creator') + display_queue = admin_link('queue') + display_owner = admin_link('owner') + updated = admin_date('updated_at') + display_state = admin_colored('state', colors=STATE_COLORS, bold=False) + display_priority = admin_colored('priority', colors=PRIORITY_COLORS, bold=False) + + @mark_safe + def display_summary(self, ticket): + context = { + 'creator': admin_link('creator')(self, ticket) if ticket.creator else ticket.creator_name, + 'created': admin_date('created_at')(ticket), + 'updated': '', + } + msg = ticket.messages.last() + if msg: + context.update({ + 'updated': admin_date('created_at')(msg), + 'updater': admin_link('author')(self, msg) if msg.author else msg.author_name, + }) + context['updated'] = '. Updated by %(updater)s about %(updated)s' % context + return '

    Added by %(creator)s about %(created)s%(updated)s

    ' % context + display_summary.short_description = 'Summary' + + def unbold_id(self, ticket): + """ Unbold id if ticket is read """ + if ticket.is_read_by(self.user): + return format_html('{}', ticket.pk) + return ticket.pk + unbold_id.short_description = "#" + unbold_id.admin_order_field = 'id' + + def bold_subject(self, ticket): + """ Bold subject when tickets are unread for request.user """ + if ticket.is_read_by(self.user): + return ticket.subject + return format_html("{}", ticket.subject) + bold_subject.short_description = _("Subject") + bold_subject.admin_order_field = 'subject' + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'subject': + kwargs['widget'] = forms.TextInput(attrs={'size':'120'}) + return super(TicketAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + def save_model(self, request, obj, *args, **kwargs): + """ Define creator for new tickets """ + if not obj.pk: + obj.creator = request.user + super(TicketAdmin, self).save_model(request, obj, *args, **kwargs) + obj.mark_as_read_by(request.user) + + def get_urls(self): + """ add markdown preview url """ + return [ + url(r'^preview/$', + wrap_admin_view(self, self.message_preview_view)) + ] + super(TicketAdmin, self).get_urls() + + def add_view(self, request, form_url='', extra_context=None): + """ Do not sow message inlines """ + return super(TicketAdmin, self).add_view(request, form_url, extra_context) + + def change_view(self, request, object_id, form_url='', extra_context=None): + """ Change view actions based on ticket state """ + ticket = get_object_or_404(Ticket, pk=object_id) + # Change view actions based on ticket state + self.change_view_actions = filter_actions(self, ticket, request) + if request.method == 'POST': + # Hack: Include the ticket changes on the request.POST + # other approaches get really messy + changes = get_ticket_changes(self, request, ticket) + if changes: + content = markdown_formated_changes(changes) + content += request.POST['messages-2-0-content'] + request.POST['messages-2-0-content'] = content + ticket.mark_as_read_by(request.user) + context = {'title': "Issue #%i - %s" % (ticket.id, ticket.subject)} + context.update(extra_context or {}) + return super(TicketAdmin, self).change_view(request, object_id, form_url=form_url, + extra_context=context) + + def changelist_view(self, request, extra_context=None): + # Hook user for bold_subject + self.user = request.user + return super(TicketAdmin,self).changelist_view(request, extra_context=extra_context) + + def message_preview_view(self, request): + """ markdown preview render via ajax """ + data = request.POST.get("data") + data_formated = markdown(strip_tags(data)) + return HttpResponse(data_formated) + + +class QueueAdmin(admin.ModelAdmin): + list_display = ('name', 'default', 'num_tickets') + actions = (set_default_queue,) + inlines = (TicketInline,) + ordering = ('name',) + + class Media: + css = { + 'all': ('orchestra/css/hide-inline-id.css',) + } + + def num_tickets(self, queue): + num = queue.tickets__count + url = reverse('admin:issues_ticket_changelist') + url += '?queue=%i' % queue.pk + return format_html('{}', url, num) + num_tickets.short_description = _("Tickets") + num_tickets.admin_order_field = 'tickets__count' + + def get_list_display(self, request): + """ show notifications """ + list_display = list(self.list_display) + for value, verbose in Contact.EMAIL_USAGES: + def display_notify(queue, notify=value): + return notify in queue.notify + display_notify.short_description = verbose + display_notify.boolean = True + list_display.append(display_notify) + return list_display + + def get_queryset(self, request): + qs = super(QueueAdmin, self).get_queryset(request) + qs = qs.annotate(models.Count('tickets')) + return qs + + +admin.site.register(Ticket, TicketAdmin) +admin.site.register(Queue, QueueAdmin) diff --git a/orchestra/contrib/issues/api.py b/orchestra/contrib/issues/api.py new file mode 100644 index 00000000..bbc5d5ee --- /dev/null +++ b/orchestra/contrib/issues/api.py @@ -0,0 +1,44 @@ +from rest_framework import viewsets, mixins +from rest_framework.decorators import action +from rest_framework.response import Response + +from orchestra.api import router, LogApiMixin + +from .models import Ticket, Queue +from .serializers import TicketSerializer, QueueSerializer + + + +class TicketViewSet(LogApiMixin, viewsets.ModelViewSet): + queryset = Ticket.objects.all() + serializer_class = TicketSerializer + + @action(detail=True) + def mark_as_read(self, request, pk=None): + ticket = self.get_object() + ticket.mark_as_read_by(request.user) + return Response({'status': 'Ticket marked as read'}) + + @action(detail=True) + def mark_as_unread(self, request, pk=None): + ticket = self.get_object() + ticket.mark_as_unread_by(request.user) + return Response({'status': 'Ticket marked as unread'}) + + def get_queryset(self): + qs = super(TicketViewSet, self).get_queryset() + qs = qs.select_related('creator', 'queue') + qs = qs.prefetch_related('messages__author') + return qs.filter(creator=self.request.user) + + +class QueueViewSet(LogApiMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet): + queryset = Queue.objects.all() + serializer_class = QueueSerializer + + +router.register(r'tickets', TicketViewSet) +router.register(r'ticket-queues', QueueViewSet) diff --git a/orchestra/contrib/issues/apps.py b/orchestra/contrib/issues/apps.py new file mode 100644 index 00000000..c2c32def --- /dev/null +++ b/orchestra/contrib/issues/apps.py @@ -0,0 +1,15 @@ +from django.apps import AppConfig + +from orchestra.core import accounts, administration +from orchestra.core.translations import ModelTranslation + + +class IssuesConfig(AppConfig): + name = 'orchestra.contrib.issues' + verbose_name = "Issues" + + def ready(self): + from .models import Queue, Ticket + accounts.register(Ticket, icon='Ticket_star.png') + administration.register(Queue, dashboard=False) + ModelTranslation.register(Queue, ('verbose_name',)) diff --git a/orchestra/contrib/issues/filters.py b/orchestra/contrib/issues/filters.py new file mode 100644 index 00000000..d0431feb --- /dev/null +++ b/orchestra/contrib/issues/filters.py @@ -0,0 +1,49 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + +from .models import Ticket + + +class MyTicketsListFilter(SimpleListFilter): + """ Filter tickets by created_by according to request.user """ + title = 'Tickets' + parameter_name = 'my_tickets' + + def lookups(self, request, model_admin): + return ( + ('True', _("My Tickets")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.involved_by(request.user) + + +class TicketStateListFilter(SimpleListFilter): + title = 'State' + parameter_name = 'state' + + def lookups(self, request, model_admin): + return ( + ('OPEN', _("Open")), + (Ticket.NEW, _("New")), + (Ticket.IN_PROGRESS, _("In Progress")), + (Ticket.RESOLVED, _("Resolved")), + (Ticket.FEEDBACK, _("Feedback")), + (Ticket.REJECTED, _("Rejected")), + (Ticket.CLOSED, _("Closed")), + ('False', _("All")), + ) + + def queryset(self, request, queryset): + if self.value() == 'OPEN': + return queryset.exclude(state__in=[Ticket.CLOSED, Ticket.REJECTED]) + elif self.value() == 'False': + return queryset + return queryset.filter(state=self.value()) + + def choices(self, cl): + """ Remove default All """ + choices = iter(super(TicketStateListFilter, self).choices(cl)) + next(choices) + return choices diff --git a/orchestra/contrib/issues/forms.py b/orchestra/contrib/issues/forms.py new file mode 100644 index 00000000..137e7097 --- /dev/null +++ b/orchestra/contrib/issues/forms.py @@ -0,0 +1,112 @@ +from django import forms +from django.contrib.auth import get_user_model +from django.utils.html import strip_tags +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ +from django.templatetags.static import static +from markdown import markdown + +from orchestra.forms.widgets import SpanWidget + +from .models import Queue, Ticket + + +class MarkDownWidget(forms.Textarea): + """ MarkDown textarea widget with syntax preview """ + + markdown_url = static('issues/markdown_syntax.html') + markdown_help_text = ( + 'markdown format' % (markdown_url, markdown_url) + ) + markdown_help_text = 'HTML not allowed, you can use %s' % markdown_help_text + + def render(self, name, value, attrs, renderer=None): + widget_id = attrs['id'] if attrs and 'id' in attrs else 'id_%s' % name + textarea = super(MarkDownWidget, self).render(name, value, attrs) + preview = ('preview'\ + '
    '.format(widget_id)) + return mark_safe('

    %s
    %s
    %s

    ' % ( + self.markdown_help_text, textarea, preview)) + + +class MessageInlineForm(forms.ModelForm): + """ Add message form """ + created_on = forms.CharField(label="Created On", required=False) + content = forms.CharField(widget=MarkDownWidget(), required=False) + + class Meta: + fields = ('author', 'author_name', 'created_on', 'content') + + def __init__(self, *args, **kwargs): + super(MessageInlineForm, self).__init__(*args, **kwargs) + self.fields['created_on'].widget = SpanWidget(display='') + + def clean_content(self): + """ clean HTML tags """ + return strip_tags(self.cleaned_data['content']) + + def save(self, *args, **kwargs): + if self.instance.pk is None: + self.instance.author = self.user + return super(MessageInlineForm, self).save(*args, **kwargs) + + +class UsersIterator(forms.models.ModelChoiceIterator): + """ Group ticket owner by superusers, ticket.group and regular users """ + def __init__(self, *args, **kwargs): + self.ticket = kwargs.pop('ticket', False) + super(forms.models.ModelChoiceIterator, self).__init__(*args, **kwargs) + + def __iter__(self): + yield ('', '---------') + users = get_user_model().objects.exclude(is_active=False).order_by('name') + superusers = users.filter(is_superuser=True) + if superusers: + yield ('Operators', list(superusers.values_list('pk', 'name'))) + users = users.exclude(is_superuser=True) + if users: + yield ('Other', list(users.values_list('pk', 'name'))) + + +class TicketForm(forms.ModelForm): + display_description = forms.CharField(label=_("Description"), required=False) + description = forms.CharField(widget=MarkDownWidget(attrs={'class':'vLargeTextField'})) + + class Meta: + model = Ticket + fields = ( + 'creator', 'creator_name', 'owner', 'queue', 'subject', 'description', + 'priority', 'state', 'cc', 'display_description' + ) + + def __init__(self, *args, **kwargs): + super(TicketForm, self).__init__(*args, **kwargs) + ticket = kwargs.get('instance', False) + users = self.fields['owner'].queryset + self.fields['owner'].queryset = users.filter(is_superuser=True) + if not ticket: + # Provide default ticket queue for new ticket + try: + self.initial['queue'] = Queue.objects.get(default=True).id + except Queue.DoesNotExist: + pass + else: + description = markdown(ticket.description) + # some hacks for better line breaking + description = description.replace('>\n', '#Ha9G9-?8') + description = description.replace('\n', '
    ') + description = description.replace('#Ha9G9-?8', '>\n') + description = '
    %s
    ' % description + widget = SpanWidget(display=description) + self.fields['display_description'].widget = widget + + def clean_description(self): + """ clean HTML tags """ + return strip_tags(self.cleaned_data['description']) + + +class ChangeReasonForm(forms.Form): + reason = forms.CharField(widget=forms.Textarea(attrs={'cols': '100', 'rows': '10'}), + required=False) diff --git a/orchestra/contrib/issues/helpers.py b/orchestra/contrib/issues/helpers.py new file mode 100644 index 00000000..a7cceb30 --- /dev/null +++ b/orchestra/contrib/issues/helpers.py @@ -0,0 +1,40 @@ +def filter_actions(modeladmin, ticket, request): + if not hasattr(modeladmin, 'change_view_actions_backup'): + modeladmin.change_view_actions_backup = list(modeladmin.change_view_actions) + actions = modeladmin.change_view_actions_backup + if ticket.state == modeladmin.model.CLOSED: + del_actions = actions + else: + from .actions import action_map + del_actions = [action_map.get(ticket.state, None)] + if ticket.owner == request.user: + del_actions.append('take') + exclude = lambda a: not (a == action or a.url_name == action) + for action in del_actions: + actions = list(filter(exclude, actions)) + return actions + + +def markdown_formated_changes(changes): + markdown = '' + for name, values in changes.items(): + context = (name.capitalize(), values[0], values[1]) + markdown += '* **%s** changed from _%s_ to _%s_\n' % context + return markdown + '\n' + + +def get_ticket_changes(modeladmin, request, ticket): + ModelForm = modeladmin.get_form(request, ticket) + form = ModelForm(request.POST, request.FILES) + changes = {} + if form.is_valid(): + for attr in ['state', 'priority', 'owner', 'queue']: + old_value = getattr(ticket, attr) + new_value = form.cleaned_data[attr] + if old_value != new_value: + choices = dict(form.fields[attr].choices) + if old_value in choices: + old_value = choices[old_value] + new_value = choices[new_value] + changes[attr] = (old_value, new_value) + return changes diff --git a/orchestra/contrib/issues/models.py b/orchestra/contrib/issues/models.py new file mode 100644 index 00000000..717f3acf --- /dev/null +++ b/orchestra/contrib/issues/models.py @@ -0,0 +1,202 @@ +from django.conf import settings as djsettings +from django.db import models +from django.db.models import query, Q +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.contacts import settings as contacts_settings +from orchestra.contrib.contacts.models import Contact +from orchestra.models.fields import MultiSelectField +from orchestra.utils.mail import send_email_template + +from . import settings + + +class Queue(models.Model): + name = models.CharField(_("name"), max_length=128, unique=True) + verbose_name = models.CharField(_("verbose_name"), max_length=128, blank=True) + default = models.BooleanField(_("default"), default=False) + notify = MultiSelectField(_("notify"), max_length=256, blank=True, + choices=Contact.EMAIL_USAGES, + default=contacts_settings.CONTACTS_DEFAULT_EMAIL_USAGES, + help_text=_("Contacts to notify by email")) + + def __str__(self): + return self.verbose_name or self.name + + def save(self, *args, **kwargs): + """ mark as default queue if needed """ + existing_default = Queue.objects.filter(default=True) + if self.default: + existing_default.update(default=False) + elif not existing_default: + self.default = True + super(Queue, self).save(*args, **kwargs) + + +class TicketQuerySet(query.QuerySet): + def involved_by(self, user, *args, **kwargs): + qset = Q(creator=user) | Q(owner=user) | Q(messages__author=user) + return self.filter(qset, *args, **kwargs).distinct() + + +class Ticket(models.Model): + HIGH = 'HIGH' + MEDIUM = 'MEDIUM' + LOW = 'LOW' + PRIORITIES = ( + (HIGH, 'High'), + (MEDIUM, 'Medium'), + (LOW, 'Low'), + ) + + NEW = 'NEW' + IN_PROGRESS = 'IN_PROGRESS' + RESOLVED = 'RESOLVED' + FEEDBACK = 'FEEDBACK' + REJECTED = 'REJECTED' + CLOSED = 'CLOSED' + STATES = ( + (NEW, 'New'), + (IN_PROGRESS, 'In Progress'), + (RESOLVED, 'Resolved'), + (FEEDBACK, 'Feedback'), + (REJECTED, 'Rejected'), + (CLOSED, 'Closed'), + ) + + creator = models.ForeignKey(djsettings.AUTH_USER_MODEL, verbose_name=_("created by"), + related_name='tickets_created', null=True, on_delete=models.SET_NULL) + creator_name = models.CharField(_("creator name"), max_length=256, blank=True) + owner = models.ForeignKey(djsettings.AUTH_USER_MODEL, null=True, blank=True, + on_delete=models.SET_NULL, + related_name='tickets_owned', verbose_name=_("assigned to")) + queue = models.ForeignKey(Queue, related_name='tickets', null=True, blank=True, + on_delete=models.SET_NULL) + subject = models.CharField(_("subject"), max_length=256) + description = models.TextField(_("description")) + priority = models.CharField(_("priority"), max_length=32, choices=PRIORITIES, default=MEDIUM) + state = models.CharField(_("state"), max_length=32, choices=STATES, default=NEW) + created_at = models.DateTimeField(_("created"), auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(_("modified"), auto_now=True) + cc = models.TextField("CC", help_text=_("emails to send a carbon copy to"), blank=True) + + objects = TicketQuerySet.as_manager() + + class Meta: + ordering = ['-updated_at'] + + def __str__(self): + return str(self.pk) + + def get_notification_emails(self): + """ Get emails of the users related to the ticket """ + emails = list(settings.ISSUES_SUPPORT_EMAILS) + emails.append(self.creator.email) + if self.owner: + emails.append(self.owner.email) + for contact in self.creator.contacts.all(): + if self.queue and set(contact.email_usage).union(set(self.queue.notify)): + emails.append(contact.email) + for message in self.messages.distinct('author'): + emails.append(message.author.email) + return set(emails + self.get_cc_emails()) + + def notify(self, message=None, content=None): + """ Send an email to ticket stakeholders notifying an state update """ + emails = self.get_notification_emails() + template = 'issues/ticket_notification.mail' + html_template = 'issues/ticket_notification_html.mail' + context = { + 'ticket': self, + 'ticket_message': message + } + send_email_template(template, context, emails, html=html_template) + + def save(self, *args, **kwargs): + """ notify stakeholders of new ticket """ + new_issue = not self.pk + if not self.creator_name and self.creator: + self.creator_name = self.creator.get_full_name() + super(Ticket, self).save(*args, **kwargs) + if new_issue: + # PK should be available for rendering the template + self.notify() + + def is_involved_by(self, user): + """ returns whether user has participated or is referenced on the ticket + as owner or member of the group + """ + return Ticket.objects.filter(pk=self.pk).involved_by(user).exists() + + def get_cc_emails(self): + return self.cc.split(',') if self.cc else [] + + def mark_as_read_by(self, user): + self.trackers.get_or_create(user=user) + + def mark_as_unread_by(self, user): + self.trackers.filter(user=user).delete() + + def mark_as_unread(self): + self.trackers.all().delete() + + def is_read_by(self, user): + return self.trackers.filter(user=user).exists() + + def reject(self): + self.state = Ticket.REJECTED + self.save(update_fields=('state', 'updated_at')) + + def resolve(self): + self.state = Ticket.RESOLVED + self.save(update_fields=('state', 'updated_at')) + + def close(self): + self.state = Ticket.CLOSED + self.save(update_fields=('state', 'updated_at')) + + def take(self, user): + self.owner = user + self.save(update_fields=('state', 'updated_at')) + + +class Message(models.Model): + ticket = models.ForeignKey('issues.Ticket', on_delete=models.CASCADE, + verbose_name=_("ticket"), related_name='messages') + author = models.ForeignKey(djsettings.AUTH_USER_MODEL, on_delete=models.CASCADE, + verbose_name=_("author"), related_name='ticket_messages') + author_name = models.CharField(_("author name"), max_length=256, blank=True) + content = models.TextField(_("content")) + created_at = models.DateTimeField(_("created at"), auto_now_add=True) + + class Meta: + get_latest_by = 'id' + + def __str__(self): + return "#%i" % self.id + + def save(self, *args, **kwargs): + """ notify stakeholders of ticket update """ + if not self.pk: + self.ticket.mark_as_unread() + self.ticket.mark_as_read_by(self.author) + self.ticket.notify(message=self) + self.author_name = self.author.get_full_name() + super(Message, self).save(*args, **kwargs) + + @property + def number(self): + return self.ticket.messages.filter(id__lte=self.id).count() + + +class TicketTracker(models.Model): + """ Keeps track of user read tickets """ + ticket = models.ForeignKey(Ticket, on_delete=models.CASCADE, + verbose_name=_("ticket"), related_name='trackers') + user = models.ForeignKey(djsettings.AUTH_USER_MODEL, on_delete=models.CASCADE, + verbose_name=_("user"), related_name='ticket_trackers') + + class Meta: + unique_together = ( + ('ticket', 'user'), + ) diff --git a/orchestra/contrib/issues/serializers.py b/orchestra/contrib/issues/serializers.py new file mode 100644 index 00000000..eb636c49 --- /dev/null +++ b/orchestra/contrib/issues/serializers.py @@ -0,0 +1,45 @@ +from rest_framework import serializers + +from .models import Ticket, Message, Queue + + +class QueueSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Queue + fields = ('url', 'id', 'name', 'default', 'notify') + read_only_fields = ('name', 'default', 'notify') + + +class MessageSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Message + fields = ('id', 'author', 'author_name', 'content', 'created_at') + read_only_fields = ('author', 'author_name', 'created_at') + + def get_identity(self, data): + return data.get('id') + + def create(self, validated_data): + validated_data['author'] = self.context['request'].user + return super(MessageSerializer, self).create(validated_data) + + +class TicketSerializer(serializers.HyperlinkedModelSerializer): + """ Validates if this zone generates a correct zone file """ + messages = MessageSerializer(required=False, many=True, read_only=True) + is_read = serializers.SerializerMethodField() + + class Meta: + model = Ticket + fields = ( + 'url', 'id', 'creator', 'creator_name', 'owner', 'queue', 'subject', + 'description', 'state', 'messages', 'is_read' + ) + read_only_fields = ('creator', 'creator_name', 'owner') + + def get_is_read(self, obj): + return obj.is_read_by(self.context['request'].user) + + def create(self, validated_data): + validated_data['creator'] = self.context['request'].user + return super(TicketSerializer, self).create(validated_data) diff --git a/orchestra/contrib/issues/settings.py b/orchestra/contrib/issues/settings.py new file mode 100644 index 00000000..902ba337 --- /dev/null +++ b/orchestra/contrib/issues/settings.py @@ -0,0 +1,16 @@ +from django.core.validators import validate_email + +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL + + +ISSUES_SUPPORT_EMAILS = Setting('ISSUES_SUPPORT_EMAILS', + (ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL,), + validators=[lambda emails: [validate_email(e) for e in emails]], + help_text="Includes ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL by default", +) + + +ISSUES_NOTIFY_SUPERUSERS = Setting('ISSUES_NOTIFY_SUPERUSERS', + True +) diff --git a/orchestra/contrib/issues/static/issues/css/ticket-admin.css b/orchestra/contrib/issues/static/issues/css/ticket-admin.css new file mode 100644 index 00000000..b52da12a --- /dev/null +++ b/orchestra/contrib/issues/static/issues/css/ticket-admin.css @@ -0,0 +1,67 @@ +fieldset .field-box { + float: left; + margin-right: 20px; + width: 300px; +} + +hr { + background-color: #B6B6B6; +} + +h4 { + color: #666; +} + +form .field-display_description p, form .field-display_description ul { + margin-left: 0; + padding-left: 12px; +} + +form .field-display_description ul { + margin-left: 24px; +} + +ul li { + list-style-type: disc; + padding: 0; +} + +/*** messages format ***/ +#messages-group { + margin-bottom: 0; +} + +#messages-2-group { + margin-top: 0; +} + +#messages-2-group h2, #messages-2-group thead { + display: none; +} + +#id_messages-2-0-content { + width: 99%; +} + +/** ticket.description preview CSS overrides **/ +.content-preview { + border: 1px solid #ccc; + padding: 2px 5px; +} + +.aligned .content-preview p { + margin-left: 5px; + padding-left: 0; +} +.module .content-preview ol, +.module .content-preview ul { + margin-left: 5px; +} + +/** unread messages admin changelist **/ +strong.unread { + display: inline-block; + padding-left: 21px; + background: url(../images/unread_ticket.gif) no-repeat left; +} + diff --git a/orchestra/contrib/issues/static/issues/images/btn_edit.gif b/orchestra/contrib/issues/static/issues/images/btn_edit.gif new file mode 100644 index 00000000..1a6f83c5 Binary files /dev/null and b/orchestra/contrib/issues/static/issues/images/btn_edit.gif differ diff --git a/orchestra/contrib/issues/static/issues/images/unread_ticket.gif b/orchestra/contrib/issues/static/issues/images/unread_ticket.gif new file mode 100644 index 00000000..62bc6ffd Binary files /dev/null and b/orchestra/contrib/issues/static/issues/images/unread_ticket.gif differ diff --git a/orchestra/contrib/issues/static/issues/js/admin-ticket.js b/orchestra/contrib/issues/static/issues/js/admin-ticket.js new file mode 100644 index 00000000..b9d392d0 --- /dev/null +++ b/orchestra/contrib/issues/static/issues/js/admin-ticket.js @@ -0,0 +1,16 @@ +(function($) { + $(document).ready(function($) { + // load markdown preview + $('.load-preview').on("click", function() { + var field = '#' + $(this).attr('data-field'), + data = { + 'data': $(field).val(), + 'csrfmiddlewaretoken': $('[name=csrfmiddlewaretoken]', + '#ticket_form').val(), + }, + preview = field + '-preview'; + $(preview).load("/admin/issues/ticket/preview/", data); + return false; + }); + }); +})(django.jQuery); diff --git a/orchestra/contrib/issues/static/issues/js/ticket-admin.js b/orchestra/contrib/issues/static/issues/js/ticket-admin.js new file mode 100644 index 00000000..21f72e48 --- /dev/null +++ b/orchestra/contrib/issues/static/issues/js/ticket-admin.js @@ -0,0 +1,30 @@ + +(function($) { + $(document).ready(function($) { + // visibility helper show on hover + $v = $('#id_visibility'); + $v_help = $('#ticket_form .field-box.field-visibility .help') + $v.hover( + function() { $v_help.show(); }, + function() { $v_help.hide(); } + ); + + // show subject edit field on click + $('#subject-edit').click(function() { + $('.field-box.field-subject').show(); + }); + + // load markdown preview + $('.load-preview').on("click", function() { + var field = '#' + $(this).attr('data-field'), + data = { + 'data': $(field).val(), + 'csrfmiddlewaretoken': $('[name=csrfmiddlewaretoken]', + '#ticket_form').val(), + }, + preview = field + '-preview'; + $(preview).load("/admin/issues/ticket/preview/", data); + return false; + }); + }); +})(django.jQuery); diff --git a/orchestra/contrib/issues/static/issues/markdown_syntax.html b/orchestra/contrib/issues/static/issues/markdown_syntax.html new file mode 100644 index 00000000..04aad231 --- /dev/null +++ b/orchestra/contrib/issues/static/issues/markdown_syntax.html @@ -0,0 +1,55 @@ + + + + + +Markdown formatting + + + + +

    Markdown Syntax Quick Reference

    + + + + + + + + + + + + + + + + + + + + + + + + + +
    Font Styles
    **Strong**Strong
    _Italic_Italic
    > QuoteQuote
         4 or more spacesCode block
    Break Lines
    end a line with 2 or more spaces  first line
    new line
    type an empty line
     
    (or containing only spaces)    		first line
    new line
    Lists
    * Item 1
    * Item 2
    • Item 1
    • Item 2
    1. Item 1
    2. Item 2
    1. Item 1
    2. Item 2
    Headings
    # Title 1 #

    Title 1

    ## Title ##

    Title 2

    Links
    <http://foo.bar>http://foo.bar
    [link](http://foo.bar/)link
    [relative link](/about/)relative link
    + +

    + Full reference of markdown syntax. +

    + + + diff --git a/orchestra/contrib/issues/templates/issues/ticket_notification.mail b/orchestra/contrib/issues/templates/issues/ticket_notification.mail new file mode 100644 index 00000000..c9b39e15 --- /dev/null +++ b/orchestra/contrib/issues/templates/issues/ticket_notification.mail @@ -0,0 +1,36 @@ +{% if subject %} +{% if not ticket_message %} +[{{ site.name }} - Issue #{{ ticket.pk }}] ({{ ticket.get_state_display }}) {{ ticket.subject }} +{% else %} +[{{ site.name }} - Issue #{{ ticket.pk }}] {% if '**State** changed' in ticket_message.content %}({{ ticket.get_state_display }}) {% endif %}{{ ticket.subject }} +{% endif %} +{% endif %} + +{% if message %} +{% if not ticket_message %} +Issue #{{ ticket.id }} has been reported by {{ ticket.created_by }}. +{% else %} +Issue #{{ ticket.id }} has been updated by {{ ticket_message.author }}. +{% autoescape off %} +{{ ticket_message.content }} +{% endautoescape %} +{% endif %} +----------------------------------------------------------------- +Issue #{{ ticket.pk }}: {{ ticket.subject }} + + * Author: {{ ticket.creator_name }} + * Status: {{ ticket.get_state_display }} + * Priority: {{ ticket.get_priority_display }} + * Visibility: {{ ticket.get_visibility_display }} + * Group: {% if ticket.group %}{{ ticket.group }}{% endif %} + * Assigned to: {% if ticket.owner %}{{ ticket.owner }}{% endif %} + * Queue: {{ ticket.queue }} + +{% autoescape off %} +{{ ticket.description }} +{% endautoescape %} +----------------------------------------------------------------- +You have received this notification because you have either subscribed to it, or are involved in it. +To change your notification preferences, please visit: {{ site.scheme }}://{{ site.domain }}{% url 'admin:issues_ticket_change' ticket.id %} +{% endif %} + diff --git a/orchestra/contrib/issues/templates/issues/ticket_notification_html.mail b/orchestra/contrib/issues/templates/issues/ticket_notification_html.mail new file mode 100644 index 00000000..a1bf3229 --- /dev/null +++ b/orchestra/contrib/issues/templates/issues/ticket_notification_html.mail @@ -0,0 +1,60 @@ +{% load markdown %} + +{% if message %} + + + + + +{% if not ticket_message %} +Issue #{{ ticket.id }} has been reported by {{ ticket.created_by }}. +{% else %} +Issue #{{ ticket.id }} has been updated by {{ ticket_message.author }}. +{% autoescape off %} +{{ ticket_message.content|markdown }} +{% endautoescape %} +{% endif %} +
    +

    Issue #{{ ticket.pk }}: {{ ticket.subject }}

    + +
      +
    • Author: {{ ticket.creator_name }}
      • +
    • Status: {{ ticket.get_state_display }}
      • +
    • Priority: {{ ticket.get_priority_display }}
      • +
    • Visibility: {{ ticket.get_visibility_display }}
      • +
    • Group: {% if ticket.group %}{{ ticket.group }}{% endif %}
      • +
    • Assigned to: {% if ticket.owner %}{{ ticket.owner }}{% endif %}
      • +
    • Queue: {{ ticket.queue }}
      • +
    +{% autoescape off %} +{{ ticket.description|markdown }} +{% endautoescape %} +
    +

    You have received this notification because you have either subscribed to it, or are involved in it.
    +To change your notification preferences, please click here: {{ site.scheme }}://{{ site.domain }}{% url 'admin:issues_ticket_change' ticket.id %}

     + + +{% endif %} diff --git a/orchestra/contrib/issues/tests.py b/orchestra/contrib/issues/tests.py new file mode 100644 index 00000000..501deb77 --- /dev/null +++ b/orchestra/contrib/issues/tests.py @@ -0,0 +1,16 @@ +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +Replace this with more appropriate tests for your application. +""" + +from django.test import TestCase + + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.assertEqual(1 + 1, 2) diff --git a/orchestra/contrib/letsencrypt/actions.py b/orchestra/contrib/letsencrypt/actions.py new file mode 100644 index 00000000..375933ac --- /dev/null +++ b/orchestra/contrib/letsencrypt/actions.py @@ -0,0 +1,115 @@ +from django.contrib import messages, admin +from django.template.response import TemplateResponse +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext, gettext_lazy as _ + +from orchestra.admin.utils import admin_link +from orchestra.contrib.orchestration import Operation, helpers + +from .helpers import is_valid_domain, read_live_lineages, configure_cert +from .forms import LetsEncryptForm + + +def letsencrypt(modeladmin, request, queryset): + wildcards = set() + domains = set() + content_error = '' + contentless = queryset.exclude(content__path='/').distinct() + if contentless: + content_error = ngettext( + gettext("Selected website %s doesn't have a webapp mounted on /."), + gettext("Selected websites %s don't have a webapp mounted on /."), + len(contentless), + ) + content_error += gettext("
    Websites need a webapp (e.g. static) mounted on / " + "for let's encrypt HTTP-01 challenge to work.") + content_error = content_error % ', '.join((admin_link()(website) for website in contentless)) + content_error = '
    • %s
    ' % content_error + queryset = queryset.prefetch_related('domains') + for website in queryset: + for domain in website.domains.all(): + if domain.name.startswith('*.'): + wildcards.add(domain.name) + else: + domains.add(domain.name) + form = LetsEncryptForm(domains, wildcards, initial={'domains': '\n'.join(domains)}) + action_value = 'letsencrypt' + if request.POST.get('post') == 'generic_confirmation': + form = LetsEncryptForm(domains, wildcards, request.POST) + if not content_error and form.is_valid(): + cleaned_data = form.cleaned_data + domains = set(cleaned_data['domains']) + operations = [] + for website in queryset: + website_domains = [d.name for d in website.domains.all()] + encrypt_domains = set() + for domain in domains: + if is_valid_domain(domain, website_domains, wildcards): + encrypt_domains.add(domain) + website.encrypt_domains = encrypt_domains + operations.extend(Operation.create_for_action(website, 'encrypt')) + modeladmin.log_change(request, website, _("Encrypted!")) + if not operations: + messages.error(request, _("No backend operation has been executed.")) + else: + logs = Operation.execute(operations) + helpers.message_user(request, logs) + live_lineages = read_live_lineages(logs) + errors = 0 + successes = 0 + no_https = 0 + for website in queryset: + try: + configure_cert(website, live_lineages) + except LookupError: + errors += 1 + messages.error(request, _("No lineage found for website %s") % website.name) + else: + if website.protocol == website.HTTP: + no_https += 1 + website.save(update_fields=('name',)) + successes += 1 + context = { + 'name': website.name, + 'errors': errors, + 'successes': successes, + 'no_https': no_https + } + if errors: + msg = ngettext( + _("No lineages found for websites {name}."), + _("No lineages found for {errors} websites."), + errors) + messages.error(request, msg % context) + if successes: + msg = ngettext( + _("{name} website has successfully been encrypted."), + _("{successes} websites have been successfully encrypted."), + successes) + messages.success(request, msg.format(**context)) + if no_https: + msg = ngettext( + _("{name} website does not have HTTPS protocol enabled."), + _("{no_https} websites do not have HTTPS protocol enabled."), + no_https) + messages.warning(request, mark_safe(msg.format(**context))) + return + opts = modeladmin.model._meta + app_label = opts.app_label + context = { + 'title': _("Let's encrypt!"), + 'action_name': _("Encrypt"), + 'content_message': gettext("You are going to request certificates for the following domains.
    " + "This operation is safe to run multiple times, " + "existing certificates will not be regenerated. " + "Also notice that let's encrypt does not currently support wildcard certificates.") + content_error, + 'action_value': action_value, + 'queryset': queryset, + 'opts': opts, + 'obj': website if len(queryset) == 1 else None, + 'app_label': app_label, + 'action_checkbox_name': admin.helpers.ACTION_CHECKBOX_NAME, + 'form': form, + } + return TemplateResponse(request, 'admin/orchestra/generic_confirmation.html', context) +letsencrypt.short_description = "Let's encrypt!" diff --git a/orchestra/contrib/letsencrypt/admin.py b/orchestra/contrib/letsencrypt/admin.py new file mode 100644 index 00000000..1f2ae624 --- /dev/null +++ b/orchestra/contrib/letsencrypt/admin.py @@ -0,0 +1,8 @@ +from orchestra.admin.utils import insertattr +from orchestra.contrib.websites.admin import WebsiteAdmin + +from .import actions + + +insertattr(WebsiteAdmin, 'change_view_actions', actions.letsencrypt) +insertattr(WebsiteAdmin, 'actions', actions.letsencrypt) diff --git a/orchestra/contrib/letsencrypt/backends.py b/orchestra/contrib/letsencrypt/backends.py new file mode 100644 index 00000000..218c0935 --- /dev/null +++ b/orchestra/contrib/letsencrypt/backends.py @@ -0,0 +1,54 @@ +import os +import textwrap + +from orchestra.contrib.orchestration import ServiceController + +from . import settings + + +class LetsEncryptController(ServiceController): + model = 'websites.Website' + verbose_name = "Let's encrypt!" + actions = ('encrypt',) + + def prepare(self): + super().prepare() + self.cleanup = [] + context = { + 'letsencrypt_auto': settings.LETSENCRYPT_AUTO_PATH, + } + self.append(textwrap.dedent(""" + %(letsencrypt_auto)s --non-interactive --no-self-upgrade \\ + --keep --expand --agree-tos certonly --webroot \\""") % context + ) + + def encrypt(self, website): + context = self.get_context(website) + self.append(" --webroot-path %(webroot)s \\" % context) + self.append(" --email %(email)s \\" % context) + self.append(" -d %(domains)s \\" % context) + self.cleanup.append("rm -rf -- %(webroot)s/.well-known" % context) + + def commit(self): + self.append(" || exit_code=$?") + for cleanup in self.cleanup: + self.append(cleanup) + context = { + 'letsencrypt_live': os.path.normpath(settings.LETSENCRYPT_LIVE_PATH), + } + self.append(textwrap.dedent(""" + # Report back the lineages in order to infere each certificate path + echo '' + find %(letsencrypt_live)s/* -maxdepth 0 + echo ''""") % context + ) + super().commit() + + def get_context(self, website): + content = website.content_set.get(path='/') + return { + 'letsencrypt_auto': settings.LETSENCRYPT_AUTO_PATH, + 'webroot': content.webapp.get_path(), + 'email': settings.LETSENCRYPT_EMAIL or website.account.email, + 'domains': ' \\\n -d '.join(website.encrypt_domains), + } diff --git a/orchestra/contrib/letsencrypt/forms.py b/orchestra/contrib/letsencrypt/forms.py new file mode 100644 index 00000000..9d1db1dc --- /dev/null +++ b/orchestra/contrib/letsencrypt/forms.py @@ -0,0 +1,32 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.utils.translation import ngettext, gettext_lazy as _ + +from .helpers import is_valid_domain + + +class LetsEncryptForm(forms.Form): + domains = forms.CharField(widget=forms.Textarea) + + def __init__(self, domains, wildcards, *args, **kwargs): + self.domains = domains + self.wildcards = wildcards + super().__init__(*args, **kwargs) + if wildcards: + help_text = _("You can add domains maching the following wildcards: %s") + self.fields['domains'].help_text += help_text % ', '.join(wildcards) + + def clean_domains(self): + domains = self.cleaned_data['domains'].split() + cleaned_domains = set() + for domain in domains: + domain = domain.strip() + if domain not in self.domains: + domain = domain.strip() + if not is_valid_domain(domain, self.domains, self.wildcards): + raise ValidationError(_( + "%s domain is not included on selected websites, " + "nor matches with any wildcard domain.") % domain + ) + cleaned_domains.add(domain) + return cleaned_domains diff --git a/orchestra/contrib/letsencrypt/helpers.py b/orchestra/contrib/letsencrypt/helpers.py new file mode 100644 index 00000000..9577d577 --- /dev/null +++ b/orchestra/contrib/letsencrypt/helpers.py @@ -0,0 +1,48 @@ +import os + + +def is_valid_domain(domain, existing, wildcards): + if domain in existing: + return True + for wildcard in wildcards: + if domain.startswith(wildcard.lstrip('*')) and domain.count('.') == wildcard.count('.'): + return True + return False + + +def read_live_lineages(logs): + live_lineages = {} + for log in logs: + reading = False + for line in log.stdout.splitlines(): + line = line.strip() + if line == '': + break + if reading: + live_lineages[line.split('/')[-1]] = line + elif line == '': + reading = True + return live_lineages + + +def configure_cert(website, live_lineages): + for domain in website.domains.all(): + try: + path = live_lineages[domain.name] + except KeyError: + pass + else: + maps = ( + ('ssl-ca', os.path.join(path, 'chain.pem')), + ('ssl-cert', os.path.join(path, 'cert.pem')), + ('ssl-key', os.path.join(path, 'privkey.pem')), + ) + for directive, path in maps: + try: + directive = website.directives.get(name=directive) + except website.directives.model.DoesNotExist: + directive = website.directives.model(name=directive, website=website) + directive.value = path + directive.save() + return + raise LookupError("Lineage not found") diff --git a/orchestra/contrib/letsencrypt/settings.py b/orchestra/contrib/letsencrypt/settings.py new file mode 100644 index 00000000..d4ca3042 --- /dev/null +++ b/orchestra/contrib/letsencrypt/settings.py @@ -0,0 +1,17 @@ +from orchestra.contrib.settings import Setting + + +LETSENCRYPT_AUTO_PATH = Setting('LETSENCRYPT_AUTO_PATH', + '/home/httpd/letsencrypt/letsencrypt-auto' +) + + +LETSENCRYPT_LIVE_PATH = Setting('LETSENCRYPT_LIVE_PATH', + '/etc/letsencrypt/live' +) + + +LETSENCRYPT_EMAIL = Setting('LETSENCRYPT_EMAIL', + '', + help_text="Uses account.email by default", +) diff --git a/orchestra/contrib/lists/__init__.py b/orchestra/contrib/lists/__init__.py new file mode 100644 index 00000000..413f2e03 --- /dev/null +++ b/orchestra/contrib/lists/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.lists.apps.ListsConfig' diff --git a/orchestra/contrib/lists/admin.py b/orchestra/contrib/lists/admin.py new file mode 100644 index 00000000..d356d726 --- /dev/null +++ b/orchestra/contrib/lists/admin.py @@ -0,0 +1,79 @@ +from django.contrib import admin +from django.urls import re_path as url +from django.contrib.auth.admin import UserAdmin +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin +from orchestra.admin.actions import disable, enable +from orchestra.admin.utils import admin_link +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import SelectAccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter +from orchestra.forms import UserCreationForm, NonStoredUserChangeForm + +from . import settings +from .filters import HasCustomAddressListFilter +from .models import List + + +class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'name', 'address_name', 'address_domain_link', 'account_link', 'display_active' + ) + add_fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('account_link', 'name', 'is_active') + }), + (_("Address"), { + 'classes': ('wide',), + 'fields': (('address_name', 'address_domain'),) + }), + (_("Admin"), { + 'classes': ('wide',), + 'fields': ('admin_email', 'password1', 'password2'), + }), + ) + fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('account_link', 'name', 'is_active') + }), + (_("Address"), { + 'classes': ('wide',), + 'description': _("Additional address besides the default <name>@%s" + ) % settings.LISTS_DEFAULT_DOMAIN, + 'fields': (('address_name', 'address_domain'),) + }), + (_("Admin"), { + 'classes': ('wide',), + 'fields': ('password',), + }), + ) + search_fields = ('name', 'address_name', 'address_domain__name', 'account__username') + list_filter = (IsActiveListFilter, HasCustomAddressListFilter) + readonly_fields = ('account_link',) + change_readonly_fields = ('name',) + form = NonStoredUserChangeForm + add_form = UserCreationForm + list_select_related = ('account', 'address_domain',) + filter_by_account_fields = ['address_domain'] + actions = (disable, enable, list_accounts) + + address_domain_link = admin_link('address_domain', order='address_domain__name') + + def get_urls(self): + useradmin = UserAdmin(List, self.admin_site) + return [ + url(r'^(\d+)/password/$', + self.admin_site.admin_view(useradmin.user_change_password)) + ] + super(ListAdmin, self).get_urls() + + def save_model(self, request, obj, form, change): + """ set password """ + if not change: + obj.set_password(form.cleaned_data["password1"]) + super(ListAdmin, self).save_model(request, obj, form, change) + + +admin.site.register(List, ListAdmin) diff --git a/orchestra/contrib/lists/api.py b/orchestra/contrib/lists/api.py new file mode 100644 index 00000000..7dc85139 --- /dev/null +++ b/orchestra/contrib/lists/api.py @@ -0,0 +1,16 @@ +from rest_framework import viewsets + +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import List +from .serializers import ListSerializer + + +class ListViewSet(LogApiMixin, AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): + queryset = List.objects.all() + serializer_class = ListSerializer + filter_fields = ('name', 'address_domain') + + +router.register(r'lists', ListViewSet) diff --git a/orchestra/contrib/lists/apps.py b/orchestra/contrib/lists/apps.py new file mode 100644 index 00000000..f4d2b062 --- /dev/null +++ b/orchestra/contrib/lists/apps.py @@ -0,0 +1,13 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class ListsConfig(AppConfig): + name = 'orchestra.contrib.lists' + verbose_name = 'Lists' + + def ready(self): + from .models import List + services.register(List, icon='email-alter.png') + from . import signals diff --git a/orchestra/contrib/lists/backends.py b/orchestra/contrib/lists/backends.py new file mode 100644 index 00000000..b6d4dc97 --- /dev/null +++ b/orchestra/contrib/lists/backends.py @@ -0,0 +1,328 @@ +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace +from orchestra.contrib.resources import ServiceMonitor + +from . import settings +from .models import List + + +class MailmanVirtualDomainController(ServiceController): + """ + Only syncs virtualdomains used on mailman addresses + """ + verbose_name = _("Mailman virtdomain-only") + model = 'lists.List' + doc_settings = (settings, + ('LISTS_VIRTUAL_ALIAS_DOMAINS_PATH',) + ) + + def is_hosted_domain(self, domain): + """ whether or not domain MX points to this server """ + return domain.has_default_mx() + + def include_virtual_alias_domain(self, context): + domain = context['address_domain'] + if domain and self.is_hosted_domain(domain): + self.append(textwrap.dedent(""" + # Add virtual domain %(address_domain)s + [[ $(grep '^\s*%(address_domain)s\s*$' %(virtual_alias_domains)s) ]] || { + echo '%(address_domain)s' >> %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + }""") % context + ) + + def is_last_domain(self, domain): + return not List.objects.filter(address_domain=domain).exists() + + def exclude_virtual_alias_domain(self, context): + domain = context['address_domain'] + if domain and self.is_last_domain(domain): + self.append(textwrap.dedent(""" + # Remove %(address_domain)s from virtual domains + sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s\ + """) % context + ) + + def save(self, mail_list): + context = self.get_context(mail_list) + self.include_virtual_alias_domain(context) + + def delete(self, mail_list): + context = self.get_context(mail_list) + self.exclude_virtual_alias_domain(context) + + def commit(self): + context = self.get_context_files() + super(MailmanVirtualDomainController, self).commit() + + def get_context_files(self): + return { + 'virtual_alias_domains': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH, + } + + def get_context(self, mail_list): + context = self.get_context_files() + context.update({ + 'address_domain': mail_list.address_domain, + }) + return replace(context, "'", '"') + + +class MailmanController(MailmanVirtualDomainController): + """ + Mailman 2 backend based on newlist, it handles custom domains. + Includes MailmanVirtualDomainController + """ + verbose_name = "Mailman" + address_suffixes = [ + '', + '-admin', + '-bounces', + '-confirm', + '-join', + '-leave', + '-owner', + '-request', + '-subscribe', + '-unsubscribe' + ] + doc_settings = (settings, ( + 'LISTS_VIRTUAL_ALIAS_PATH', + 'LISTS_VIRTUAL_ALIAS_DOMAINS_PATH', + 'LISTS_DEFAULT_DOMAIN', + 'LISTS_MAILMAN_ROOT_DIR' + )) + + def get_virtual_aliases(self, context): + aliases = ['# %(banner)s' % context] + for suffix in self.address_suffixes: + context['suffix'] = suffix + # Because mailman doesn't properly handle lists aliases we need virtual aliases + if context['address_name'] != context['name']: + aliases.append("%(address_name)s%(suffix)s@%(domain)s\t%(name)s%(suffix)s@grups.pangea.org" % context) + return '\n'.join(aliases) + + + def save(self, mail_list): + context = self.get_context(mail_list) + + # Create list + cmd = "/opt/mailman/venv/bin/python /usr/local/admin/orchestra_mailman3/save.py %(name)s %(admin)s %(address_name)s@%(domain)s" % context + if not mail_list.active: + cmd += ' --inactive' + self.append(cmd) + + # Custom domain + if mail_list.address: + context.update({ + 'aliases': self.get_virtual_aliases(context), + 'num_entries': 2 if context['address_name'] != context['name'] else 1, + }) + self.append(textwrap.dedent("""\ + # Create list alias for custom domain + aliases='%(aliases)s' + if ! grep '\s\s*%(name)s\s*$' %(virtual_alias)s > /dev/null; then + echo "${aliases}" >> %(virtual_alias)s + UPDATED_VIRTUAL_ALIAS=1 + else + if grep -E '(%(address_name)s|%(name)s)@(%(address_domain)s|grups.pangea.org)' %(virtual_alias)s > /dev/null ; then + sed -i -e '/^.*%(name)s\(-admin\|-bounces\|-confirm\|-join\|-leave\|-owner\|-request\|-subscribe\|-unsubscribe\|@\).*$/d' \\ + -e '/# .*%(name)s$/d' %(virtual_alias)s + echo "${aliases}" >> %(virtual_alias)s + UPDATED_VIRTUAL_ALIAS=1 + fi + fi """) % context + ) + else: + self.append(textwrap.dedent("""\ + # Cleanup possible ex-custom domain + if grep '\s\s*%(name)s\s*$' %(virtual_alias)s > /dev/null; then + #sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s + sed -i -e '/^.*%(name)s\(-admin\|-bounces\|-confirm\|-join\|-leave\|-owner\|-request\|-subscribe\|-unsubscribe\|@\).*$/d' \\ + -e '/# .*%(name)s$/d' %(virtual_alias)s + fi""") % context + ) + + + def delete(self, mail_list): + context = self.get_context(mail_list) + + # Custom domain delete + self.append(textwrap.dedent("""\ + # Cleanup possible ex-custom domain + if grep '\s\s*%(name)s\s*$' %(virtual_alias)s > /dev/null; then + sed -i -e '/^.*%(name)s\(-admin\|-bounces\|-confirm\|-join\|-leave\|-owner\|-request\|-subscribe\|-unsubscribe\|@\).*$/d' \\ + -e '/# .*%(name)s$/d' %(virtual_alias)s + fi""") % context + ) + + # Delete list + cmd = "/opt/mailman/venv/bin/python /usr/local/admin/orchestra_mailman3/delete.py %(name)s" % context + self.append(cmd) + + + def commit(self): + pass + + def get_context_files(self): + return { + 'virtual_alias': settings.LISTS_VIRTUAL_ALIAS_PATH, + 'virtual_alias_domains': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH, + } + + def get_banner(self, mail_list): + banner = super(MailmanController, self).get_banner() + return '%s %s' % (banner, mail_list.name) + + def get_context(self, mail_list): + context = self.get_context_files() + context.update({ + 'banner': self.get_banner(mail_list), + 'name': mail_list.name, + 'password': mail_list.password, + 'domain': mail_list.address_domain or settings.LISTS_DEFAULT_DOMAIN, + 'address_name': mail_list.get_address_name(), + 'address_domain': mail_list.address_domain, + 'suffixes_regex': '\|'.join(self.address_suffixes), + 'admin': mail_list.admin_email, + 'mailman_root': settings.LISTS_MAILMAN_ROOT_DIR, + }) + return replace(context, "'", '"') + + +class MailmanTraffic(ServiceMonitor): + """ + Parses mailman log file looking for email size and multiples it by list_members count. + """ + model = 'lists.List' + resource = ServiceMonitor.TRAFFIC + verbose_name = _("Mailman traffic") + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + doc_settings = (settings, + ('LISTS_MAILMAN_POST_LOG_PATH',) + ) + + def prepare(self): + postlog = settings.LISTS_MAILMAN_POST_LOG_PATH + context = { + 'postlogs': str((postlog, postlog+'.1')), + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + } + self.append(textwrap.dedent("""\ + import re + import subprocess + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + postlogs = {postlogs} + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + lists = {{}} + months = {{ + 'Jan': '01', + 'Feb': '02', + 'Mar': '03', + 'Apr': '04', + 'May': '05', + 'Jun': '06', + 'Jul': '07', + 'Aug': '08', + 'Sep': '09', + 'Oct': '10', + 'Nov': '11', + 'Dec': '12', + }} + mailman_addr = re.compile(r'.*-(admin|bounces|confirm|join|leave|owner|request|subscribe|unsubscribe)@.*|mailman@.*') + + def prepare(object_id, list_name, ini_date): + global lists + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + lists[list_name] = [ini_date, object_id, 0] + + def monitor(lists, end_date, months, postlogs): + for postlog in postlogs: + try: + with open(postlog, 'r') as postlog: + for line in postlog.readlines(): + line = line.split() + if len(line) < 11: + continue + month, day, time, year, __, __, __, list_name, __, addr, size = line[:11] + try: + list = lists[list_name] + except KeyError: + continue + else: + # discard mailman messages because of inconsistent POST logging + if mailman_addr.match(addr): + continue + date = year + months[month] + day + time.replace(':', '') + if list[0] < int(date) < end_date: + size = size[5:-1] + try: + list[2] += int(size) + except ValueError: + # anonymized post + pass + except IOError as e: + sys.stderr.write(str(e)+'\\n') + + for list_name, opts in lists.items(): + __, object_id, size = opts + if size: + cmd = ' '.join(('list_members', list_name, '| wc -l')) + ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subscribers = ps.communicate()[0].strip() + size *= int(subscribers) + sys.stderr.write("%s %s*%s traffic*subscribers\\n" % (object_id, size, subscribers)) + print object_id, size + """).format(**context) + ) + + def monitor(self, user): + context = self.get_context(user) + self.append("prepare(%(object_id)s, '%(list_name)s', '%(last_date)s')" % context) + + def commit(self): + self.append('monitor(lists, end_date, months, postlogs)') + + def get_context(self, mail_list): + context = { + 'list_name': mail_list.name, + 'object_id': mail_list.pk, + 'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + return replace(context, "'", '"') + + +class MailmanSubscribers(ServiceMonitor): + """ + Monitors number of list subscribers via list_members + """ + model = 'lists.List' + verbose_name = _("Mailman subscribers") + delete_old_equal_values = True + + def monitor(self, mail_list): + context = self.get_context(mail_list) + self.append('echo %(object_id)i $(list_members %(list_name)s | wc -l)' % context) + + def get_context(self, mail_list): + context = { + 'list_name': mail_list.name, + 'object_id': mail_list.pk, + } + return replace(context, "'", '"') diff --git a/orchestra/contrib/lists/filters.py b/orchestra/contrib/lists/filters.py new file mode 100644 index 00000000..8ba06ebb --- /dev/null +++ b/orchestra/contrib/lists/filters.py @@ -0,0 +1,21 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + + +class HasCustomAddressListFilter(SimpleListFilter): + """ Filter addresses whether they have any webapp or not """ + title = _("has custom address") + parameter_name = 'has_custom_address' + + def lookups(self, request, model_admin): + return ( + ('True', _("True")), + ('False', _("False")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.exclude(address_name='') + elif self.value() == 'False': + return queryset.filter(address_name='') + return queryset diff --git a/orchestra/contrib/lists/models.py b/orchestra/contrib/lists/models.py new file mode 100644 index 00000000..8ac33723 --- /dev/null +++ b/orchestra/contrib/lists/models.py @@ -0,0 +1,85 @@ +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from orchestra.core.validators import validate_name + +from . import settings + + +class ListQuerySet(models.QuerySet): + def create(self, **kwargs): + """ Sets password if provided, all within a single DB operation """ + password = kwargs.pop('password') + instance = self.model(**kwargs) + if password: + instance.set_password(password) + instance.save() + return instance + + +# TODO address and domain, perhaps allow only domain? +class List(models.Model): + name = models.CharField(_("name"), max_length=64, unique=True, validators=[validate_name], + help_text=_("Default list address <name>@%s") % settings.LISTS_DEFAULT_DOMAIN) + address_name = models.CharField(_("address name"), max_length=64, + validators=[validate_name], blank=True) + address_domain = models.ForeignKey(settings.LISTS_DOMAIN_MODEL, on_delete=models.SET_NULL, + verbose_name=_("address domain"), blank=True, null=True) + admin_email = models.EmailField(_("admin email"), + help_text=_("Administration email address")) + account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), + related_name='lists', on_delete=models.CASCADE) + # TODO also admin + is_active = models.BooleanField(_("active"), default=True, + help_text=_("Designates whether this account should be treated as active. " + "Unselect this instead of deleting accounts.")) + password = None + + objects = ListQuerySet.as_manager() + + class Meta: + unique_together = ('address_name', 'address_domain') + + def __str__(self): + return self.name + + @property + def address(self): + if self.address_name and self.address_domain: + return "%s@%s" % (self.address_name, self.address_domain) + return '' + + @cached_property + def active(self): + return self.is_active and self.account.is_active + + def clean(self): + if self.address_name and not self.address_domain_id: + raise ValidationError({ + 'address_domain': _("Domain should be selected for provided address name."), + }) + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def get_address_name(self): + return self.address_name or self.name + + def get_username(self): + return self.name + + def set_password(self, password): + self.password = password + + def get_absolute_url(self): + context = { + 'name': self.name + } + return settings.LISTS_LIST_URL % context diff --git a/orchestra/contrib/lists/serializers.py b/orchestra/contrib/lists/serializers.py new file mode 100644 index 00000000..593612ad --- /dev/null +++ b/orchestra/contrib/lists/serializers.py @@ -0,0 +1,44 @@ +from django.core.validators import RegexValidator +from django.forms import widgets +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.api.serializers import SetPasswordHyperlinkedSerializer, RelatedHyperlinkedModelSerializer +from orchestra.contrib.accounts.serializers import AccountSerializerMixin +from orchestra.core.validators import validate_password + +from .models import List + + +class RelatedDomainSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = List.address_domain.field.related_model + fields = ('url', 'id', 'name') + + +class ListSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): + password = serializers.CharField(max_length=128, label=_('Password'), + write_only=True, style={'widget': widgets.PasswordInput}, + validators=[ + validate_password, + RegexValidator(r'^[^"\'\\]+$', + _('Enter a valid password. ' + 'This value may contain any ascii character except for ' + ' \'/"/\\/ characters.'), 'invalid'), + ]) + + address_domain = RelatedDomainSerializer(required=False) + + class Meta: + model = List + fields = ('url', 'id', 'name', 'password', 'address_name', 'address_domain', 'admin_email', 'is_active',) + postonly_fields = ('name', 'password') + + def validate_address_domain(self, address_name): + if self.instance: + address_domain = address_domain or self.instance.address_domain + address_name = address_name or self.instance.address_name + if address_name and not address_domain: + raise serializers.ValidationError( + _("address_domains should should be provided when providing an addres_name")) + return address_name diff --git a/orchestra/contrib/lists/settings.py b/orchestra/contrib/lists/settings.py new file mode 100644 index 00000000..9d5e25ae --- /dev/null +++ b/orchestra/contrib/lists/settings.py @@ -0,0 +1,40 @@ +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN + + +LISTS_DOMAIN_MODEL = Setting('LISTS_DOMAIN_MODEL', + 'domains.Domain', + validators=[Setting.validate_model_label] +) + + +LISTS_DEFAULT_DOMAIN = Setting('LISTS_DEFAULT_DOMAIN', + 'lists.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default." +) + + +LISTS_LIST_URL = Setting('LISTS_LIST_URL', + 'https://lists.{}/mailman/listinfo/%(name)s'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default." +) + + +LISTS_MAILMAN_POST_LOG_PATH = Setting('LISTS_MAILMAN_POST_LOG_PATH', + '/var/log/mailman3/smtp' +) + + +LISTS_MAILMAN_ROOT_DIR = Setting('LISTS_MAILMAN_ROOT_DIR', + '/var/lib/mailman3' +) + + +LISTS_VIRTUAL_ALIAS_PATH = Setting('LISTS_VIRTUAL_ALIAS_PATH', + '/etc/postfix/mailman3_virtusertable' +) + + +LISTS_VIRTUAL_ALIAS_DOMAINS_PATH = Setting('LISTS_VIRTUAL_ALIAS_DOMAINS_PATH', + '/etc/postfix/mailman3_virtdomains' +) diff --git a/orchestra/contrib/lists/signals.py b/orchestra/contrib/lists/signals.py new file mode 100644 index 00000000..1b2d54f9 --- /dev/null +++ b/orchestra/contrib/lists/signals.py @@ -0,0 +1,19 @@ +from django.apps import apps +from django.db.models.signals import pre_delete +from django.dispatch import receiver + +from . import settings +from .models import List + + +DOMAIN_MODEL = apps.get_model(settings.LISTS_DOMAIN_MODEL) + + +@receiver(pre_delete, sender=DOMAIN_MODEL, dispatch_uid="lists.clean_address_name") +def clean_address_name(sender, **kwargs): + domain = kwargs['instance'] + for list in List.objects.filter(address_domain_id=domain.pk): + list.address_name = '' + list.address_domain_id = None + list.save(update_fields=('address_name', 'address_domain_id')) + diff --git a/orchestra/contrib/lists/tests/__init__.py b/orchestra/contrib/lists/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/lists/tests/functional_tests/__init__.py b/orchestra/contrib/lists/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/lists/tests/functional_tests/tests.py b/orchestra/contrib/lists/tests/functional_tests/tests.py new file mode 100644 index 00000000..447a8249 --- /dev/null +++ b/orchestra/contrib/lists/tests/functional_tests/tests.py @@ -0,0 +1,278 @@ +import os +import smtplib +import time +import unittest +from email.mime.text import MIMEText + +import requests +from django.conf import settings as djsettings +from django.core.management.base import CommandError +from django.urls import reverse +from orchestra.admin.utils import change_url +from orchestra.contrib.domains.models import Domain +from orchestra.contrib.orchestration.models import Route, Server +from orchestra.utils.sys import sshrun +from orchestra.utils.tests import (BaseLiveServerTestCase, random_ascii, + save_response_on_error, snapshot_on_error) +from selenium.webdriver.support.select import Select + +from ... import backends, settings +from ...models import List + +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + + +class ListMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') + DEPENDENCIES = ( + 'orchestra.contrib.orchestration', + 'orchestra.contrib.domains', + 'orchestra.contrib.lists', + ) + + def setUp(self): + super(ListMixin, self).setUp() + self.add_route() + djsettings.DEBUG = True + + def validate_add(self, name, address=None): + sshrun(self.MASTER_SERVER, 'list_members %s' % name, display=False) + if not address: + address = "%s@%s" % (name, settings.LISTS_DEFAULT_DOMAIN) + subscribe_address = "{}-subscribe@{}".format(*address.split('@')) + request_address = "{}-request@{}".format(name, address.split('@')[1]) + self.subscribe(subscribe_address) + time.sleep(3) + sshrun(self.MASTER_SERVER, + 'grep -v ":\|^\s\|^$\|-\|\.\|\s" /var/spool/mail/nobody | base64 -d | grep "%s"' + % request_address, display=False) + + def validate_login(self, name, password): + url = 'http://%s/cgi-bin/mailman/admin/%s' % (settings.LISTS_DEFAULT_DOMAIN, name) + self.assertEqual(200, requests.post(url, data={'adminpw': password}).status_code) + + def validate_delete(self, name): + context = { + 'name': name, + 'domain': Domain.objects.get().name, + 'virtual_domain': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH, + 'virtual_alias': settings.LISTS_VIRTUAL_ALIAS_PATH, + } + self.assertRaises(CommandError, sshrun, self.MASTER_SERVER, + 'grep "\s%(name)s\s*" %(virtual_alias)s' % context, display=False) + self.assertRaises(CommandError, sshrun, self.MASTER_SERVER, + 'grep "^\s*$(domain)s\s*$" %(virtual_domain)s' % context, display=False) + self.assertRaises(CommandError, sshrun, self.MASTER_SERVER, + 'list_lists | grep -i "^\s*%(name)s\s"' % context, display=False) + + def subscribe(self, subscribe_address): + msg = MIMEText('') + msg['To'] = subscribe_address + msg['From'] = 'root@%s' % self.MASTER_SERVER + msg['Subject'] = 'subscribe' + server = smtplib.SMTP(self.MASTER_SERVER, 25) + try: + server.ehlo() + server.starttls() + server.ehlo() + server.sendmail(msg['From'], msg['To'], msg.as_string()) + finally: + server.quit() + + def add_route(self): + server = Server.objects.create(name=self.MASTER_SERVER) + backend = backends.MailmanController.get_name() + Route.objects.create(backend=backend, match=True, host=server) + + def test_add(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + admin_email = 'root@test3.orchestra.lan' + self.add(name, password, admin_email) + self.validate_add(name) + self.validate_login(name, password) + self.addCleanup(self.delete, name) + + def test_add_with_address(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + admin_email = 'root@test3.orchestra.lan' + address_name = '%s_name' % random_ascii(10) + domain_name = '%sdomain.lan' % random_ascii(10) + address_domain = Domain.objects.create(name=domain_name, account=self.account) + self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain) + self.addCleanup(self.delete, name) + # Mailman doesn't support changing the address, only the domain + self.validate_add(name, address="%s@%s" % (address_name, address_domain)) + + def test_change_password(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + admin_email = 'root@test3.orchestra.lan' + self.add(name, password, admin_email) + self.addCleanup(self.delete, name) + self.validate_login(name, password) + new_password = '@!?%spppP001' % random_ascii(5) + self.change_password(name, new_password) + self.validate_login(name, new_password) + + def test_change_domain(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + admin_email = 'root@test3.orchestra.lan' + address_name = '%s_name' % random_ascii(10) + domain_name = '%sdomain.lan' % random_ascii(10) + address_domain = Domain.objects.create(name=domain_name, account=self.account) + self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain) + self.addCleanup(self.delete, name) + # Mailman doesn't support changing the address, only the domain + domain_name = '%sdomain.lan' % random_ascii(10) + address_domain = Domain.objects.create(name=domain_name, account=self.account) + self.update_domain(name, domain_name) + self.validate_add(name, address="%s@%s" % (address_name, address_domain)) + + def test_change_address_name(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + admin_email = 'root@test3.orchestra.lan' + address_name = '%s_name' % random_ascii(10) + domain_name = '%sdomain.lan' % random_ascii(10) + address_domain = Domain.objects.create(name=domain_name, account=self.account) + self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain) + self.addCleanup(self.delete, name) + # Mailman doesn't support changing the address, only the domain + address_name = '%s_name' % random_ascii(10) + self.update_address_name(name, address_name) + self.validate_add(name, address="%s@%s" % (address_name, address_domain)) + + def test_delete(self): + name = '%s_list' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + admin_email = 'root@test3.orchestra.lan' + address_name = '%s_name' % random_ascii(10) + domain_name = '%sdomain.lan' % random_ascii(10) + address_domain = Domain.objects.create(name=domain_name, account=self.account) + self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain) + # Mailman doesn't support changing the address, only the domain + self.validate_add(name, address="%s@%s" % (address_name, address_domain)) + self.delete(name) + self.assertRaises(AssertionError, self.validate_login, name, password) + self.validate_delete(name) + + +@unittest.skipUnless(TEST_REST_API, "REST API tests") +class RESTListMixin(ListMixin): + def setUp(self): + super(RESTListMixin, self).setUp() + self.rest_login() + + @save_response_on_error + def add(self, name, password, admin_email, address_name=None, address_domain=None): + extra = {} + if address_name: + extra.update({ + 'address_name': address_name, + 'address_domain': self.rest.domains.retrieve(name=address_domain.name).get(), + }) + self.rest.lists.create(name=name, password=password, admin_email=admin_email, **extra) + + @save_response_on_error + def delete(self, name): + self.rest.lists.retrieve(name=name).delete() + + @save_response_on_error + def change_password(self, name, password): + mail_list = self.rest.lists.retrieve(name=name).get() + mail_list.set_password(password) + + @save_response_on_error + def update_domain(self, name, domain_name): + mail_list = self.rest.lists.retrieve(name=name).get() + domain = self.rest.domains.retrieve(name=domain_name).get() + mail_list.update(address_domain=domain) + + @save_response_on_error + def update_address_name(self, name, address_name): + mail_list = self.rest.lists.retrieve(name=name).get() + mail_list.update(address_name=address_name) + + +class AdminListMixin(ListMixin): + def setUp(self): + super(AdminListMixin, self).setUp() + self.admin_login() + + @snapshot_on_error + def add(self, name, password, admin_email, address_name=None, address_domain=None): + url = self.live_server_url + reverse('admin:lists_list_add') + self.selenium.get(url) + + name_field = self.selenium.find_element_by_id('id_name') + name_field.send_keys(name) + + password_field = self.selenium.find_element_by_id('id_password1') + password_field.send_keys(password) + password_field = self.selenium.find_element_by_id('id_password2') + password_field.send_keys(password) + + admin_email_field = self.selenium.find_element_by_id('id_admin_email') + admin_email_field.send_keys(admin_email) + + if address_name: + address_name_field = self.selenium.find_element_by_id('id_address_name') + address_name_field.send_keys(address_name) + + domain = Domain.objects.get(name=address_domain) + domain_input = self.selenium.find_element_by_id('id_address_domain') + domain_select = Select(domain_input) + domain_select.select_by_value(str(domain.pk)) + + name_field.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def delete(self, name): + mail_list = List.objects.get(name=name) + self.admin_delete(mail_list) + + @snapshot_on_error + def change_password(self, name, password): + mail_list = List.objects.get(name=name) + self.admin_change_password(mail_list, password) + + @snapshot_on_error + def update_domain(self, name, domain_name): + mail_list = List.objects.get(name=name) + url = self.live_server_url + change_url(mail_list) + self.selenium.get(url) + + domain = Domain.objects.get(name=domain_name) + domain_input = self.selenium.find_element_by_id('id_address_domain') + domain_select = Select(domain_input) + domain_select.select_by_value(str(domain.pk)) + + save = self.selenium.find_element_by_name('_save') + save.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def update_address_name(self, name, address_name): + mail_list = List.objects.get(name=name) + url = self.live_server_url + change_url(mail_list) + self.selenium.get(url) + + address_name_field = self.selenium.find_element_by_id('id_address_name') + address_name_field.clear() + address_name_field.send_keys(address_name) + + save = self.selenium.find_element_by_name('_save') + save.submit() + self.assertNotEqual(url, self.selenium.current_url) + + +class RESTListTest(RESTListMixin, BaseLiveServerTestCase): + pass + + +class AdminListTest(AdminListMixin, BaseLiveServerTestCase): + pass diff --git a/orchestra/contrib/mailboxes/__init__.py b/orchestra/contrib/mailboxes/__init__.py new file mode 100644 index 00000000..dbf89749 --- /dev/null +++ b/orchestra/contrib/mailboxes/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.mailboxes.apps.MailboxesConfig' diff --git a/orchestra/contrib/mailboxes/actions.py b/orchestra/contrib/mailboxes/actions.py new file mode 100644 index 00000000..bef631da --- /dev/null +++ b/orchestra/contrib/mailboxes/actions.py @@ -0,0 +1,13 @@ +from orchestra.admin.actions import SendEmail + + +class SendMailboxEmail(SendEmail): + def get_email_addresses(self): + for mailbox in self.queryset.all(): + yield mailbox.get_local_address() + + +class SendAddressEmail(SendEmail): + def get_email_addresses(self): + for address in self.queryset.all(): + yield address.email diff --git a/orchestra/contrib/mailboxes/admin.py b/orchestra/contrib/mailboxes/admin.py new file mode 100644 index 00000000..d1094d3f --- /dev/null +++ b/orchestra/contrib/mailboxes/admin.py @@ -0,0 +1,327 @@ +import copy +from urllib.parse import parse_qs + +from django import forms +from django.contrib import admin, messages +from django.urls import reverse +from django.db.models import F, Count, Value as V +from django.db.models.functions import Concat +from django.utils.html import format_html, format_html_join +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin +from orchestra.admin.actions import disable, enable +from orchestra.admin.utils import admin_link, change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import SelectAccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter +from orchestra.core import caches + +from . import settings +from .actions import SendMailboxEmail, SendAddressEmail +from .filters import HasMailboxListFilter, HasForwardListFilter, HasAddressListFilter +from .forms import MailboxCreationForm, MailboxChangeForm, AddressForm +from .models import Mailbox, Address, Autoresponse +from .widgets import OpenCustomFilteringOnSelect + + +class AutoresponseInline(admin.StackedInline): + model = Autoresponse + verbose_name_plural = _("autoresponse") + + def formfield_for_dbfield(self, db_field, **kwargs): + if db_field.name == 'subject': + kwargs['widget'] = forms.TextInput(attrs={'size':'118'}) + return super(AutoresponseInline, self).formfield_for_dbfield(db_field, **kwargs) + + +class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'name', 'account_link', 'display_filtering', 'display_addresses', 'display_active', + ) + list_filter = (IsActiveListFilter, HasAddressListFilter, 'filtering') + search_fields = ( + 'account__username', 'account__short_name', 'account__full_name', 'name', + 'addresses__name', 'addresses__domain__name', + ) + add_fieldsets = ( + (None, { + 'fields': ('account_link', 'name', 'password1', 'password2', 'filtering'), + }), + (_("Custom filtering"), { + 'classes': ('collapse',), + 'description': _("Please remember to select custom filtering " + "if you want this filter to be applied."), + 'fields': ('custom_filtering',), + }), + (_("Addresses"), { + 'fields': ('addresses',) + }), + ) + fieldsets = ( + (None, { + 'fields': ('name', 'password', 'is_active', 'account_link', 'filtering'), + }), + (_("Custom filtering"), { + 'classes': ('collapse',), + 'fields': ('custom_filtering',), + }), + (_("Addresses"), { + 'fields': ('addresses', 'display_forwards') + }), + ) + readonly_fields = ('account_link', 'display_addresses', 'display_forwards') + change_readonly_fields = ('name',) + add_form = MailboxCreationForm + form = MailboxChangeForm + list_prefetch_related = ('addresses__domain',) + actions = (disable, enable, list_accounts) + + def __init__(self, *args, **kwargs): + super(MailboxAdmin, self).__init__(*args, **kwargs) + if settings.MAILBOXES_LOCAL_DOMAIN: + type(self).actions = self.actions + (SendMailboxEmail(),) + + @mark_safe + def display_addresses(self, mailbox): + # Get from forwards + cache = caches.get_request_cache() + cached_forwards = cache.get('forwards') + if cached_forwards is None: + cached_forwards = {} + qs = Address.objects.filter(forward__regex=r'(^|.*\s)[^@]+(\s.*|$)') + qs = qs.annotate(email=Concat('name', V('@'), 'domain__name')) + qs = qs.values_list('id', 'email', 'forward') + for addr_id, email, mbox in qs: + url = reverse('admin:mailboxes_address_change', args=(addr_id,)) + link = format_html('{}', url, email) + try: + cached_forwards[mbox].append(link) + except KeyError: + cached_forwards[mbox] = [link] + cache.set('forwards', cached_forwards) + try: + forwards = cached_forwards[mailbox.name] + except KeyError: + forwards = [] + # Get from mailboxes + addresses = [] + for addr in mailbox.addresses.all(): + url = change_url(addr) + addresses.append(format_html('{}', url, addr.email)) + return '
    '.join(addresses+forwards) + display_addresses.short_description = _("Addresses") + + def display_forwards(self, mailbox): + forwards = mailbox.get_forwards() + return format_html_join( + '
    ', '{}', + [(change_url(addr), addr.email) for addr in forwards] + ) + display_forwards.short_description = _("Forward from") + + @mark_safe + def display_filtering(self, mailbox): + return mailbox.get_filtering_display() + display_filtering.short_description = _("Filtering") + display_filtering.admin_order_field = 'filtering' + + def formfield_for_dbfield(self, db_field, **kwargs): + if db_field.name == 'filtering': + kwargs['widget'] = OpenCustomFilteringOnSelect() + return super(MailboxAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + def get_fieldsets(self, request, obj=None): + fieldsets = super(MailboxAdmin, self).get_fieldsets(request, obj) + if obj and obj.filtering == obj.CUSTOM: + # not collapsed filtering when exists + fieldsets = copy.deepcopy(fieldsets) + fieldsets[1][1]['classes'] = fieldsets[0][1]['fields'] + ('collapse', 'open',) + elif '_to_field' in parse_qs(request.META['QUERY_STRING']): + # remove address from popup + fieldsets = list(copy.deepcopy(fieldsets)) + fieldsets.pop(-1) + return fieldsets + + def get_form(self, *args, **kwargs): + form = super(MailboxAdmin, self).get_form(*args, **kwargs) + form.modeladmin = self + return form + + def get_search_results(self, request, queryset, search_term): + # Remove local domain from the search term if present (implicit local addreç) + search_term = search_term.replace('@'+settings.MAILBOXES_LOCAL_DOMAIN, '') + # Split address name from domain in order to support address searching + search_term = search_term.replace('@', ' ') + return super(MailboxAdmin, self).get_search_results(request, queryset, search_term) + + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): + if not add: + self.check_unrelated_address(request, obj) + self.check_matching_address(request, obj) + return super(MailboxAdmin, self).render_change_form( + request, context, add, change, form_url, obj) + + def log_addition(self, request, object, *args, **kwargs): + self.check_unrelated_address(request, object) + self.check_matching_address(request, object) + return super(MailboxAdmin, self).log_addition(request, object, *args, **kwargs) + + def check_matching_address(self, request, obj): + local_domain = settings.MAILBOXES_LOCAL_DOMAIN + if obj.name and local_domain: + try: + addr = Address.objects.get( + name=obj.name, domain__name=local_domain, account_id=self.account.pk) + except Address.DoesNotExist: + pass + else: + if addr not in obj.addresses.all(): + msg = _("Mailbox '%s' local address matches '%s', please consider if " + "selecting it makes sense.") % (obj, addr) + if msg not in (m.message for m in messages.get_messages(request)): + self.message_user(request, msg, level=messages.WARNING) + + def check_unrelated_address(self, request, obj): + # Check if there exists an unrelated local Address for this mbox + local_domain = settings.MAILBOXES_LOCAL_DOMAIN + if local_domain and obj.name: + non_mbox_addresses = Address.objects.exclude(mailboxes__name=obj.name).exclude( + forward__regex=r'.*(^|\s)+%s($|\s)+.*' % obj.name) + try: + addr = non_mbox_addresses.get(name=obj.name, domain__name=local_domain) + except Address.DoesNotExist: + pass + else: + url = reverse('admin:mailboxes_address_change', args=(addr.pk,)) + msg = mark_safe( + _("Address {addr} clashes with '{mailbox}' mailbox " + "local address. Consider adding this mailbox to the address.").format( + mailbox=obj.name, url=url, addr=addr) + ) + # Prevent duplication (add_view+continue) + if msg not in (m.message for m in messages.get_messages(request)): + self.message_user(request, msg, level=messages.WARNING) + + def save_model(self, request, obj, form, change): + """ save hacky mailbox.addresses and local domain clashing """ + if obj.filtering != obj.CUSTOM: + msg = _("You have provided a custom filtering but filtering " + "selected option is %s") % obj.get_filtering_display() + if change: + old = Mailbox.objects.get(pk=obj.pk) + if old.custom_filtering != obj.custom_filtering: + messages.warning(request, msg) + elif obj.custom_filtering: + messages.warning(request, msg) + super(MailboxAdmin, self).save_model(request, obj, form, change) + obj.addresses.set(form.cleaned_data['addresses']) + + +class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'display_email', 'account_link', 'domain_link', 'display_mailboxes', 'display_forward', + ) + list_filter = (HasMailboxListFilter, HasForwardListFilter) + fields = ('account_link', 'email_link', 'mailboxes', 'forward', 'display_all_mailboxes') + add_fields = ('account_link', ('name', 'domain'), 'mailboxes', 'forward') +# inlines = [AutoresponseInline] + search_fields = ( + 'forward', 'mailboxes__name', 'account__username', 'computed_email', 'domain__name' + ) + readonly_fields = ('account_link', 'domain_link', 'email_link', 'display_all_mailboxes') + actions = (SendAddressEmail(),) + filter_by_account_fields = ('domain', 'mailboxes') + filter_horizontal = ['mailboxes'] + form = AddressForm + list_prefetch_related = ('mailboxes', 'domain') + + domain_link = admin_link('domain', order='domain__name') + + def display_email(self, address): + return address.computed_email + display_email.short_description = _("Email") + display_email.admin_order_field = 'computed_email' + + def email_link(self, address): + link = self.domain_link(address) + return format_html("{}@{}", address.name, link) + email_link.short_description = _("Email") + + def display_mailboxes(self, address): + boxes = address.mailboxes.all() + return format_html_join( + mark_safe('
    '), '{}', + [(change_url(mailbox), mailbox.name) for mailbox in boxes] + ) + display_mailboxes.short_description = _("Mailboxes") + display_mailboxes.admin_order_field = 'mailboxes__count' + + def display_all_mailboxes(self, address): + boxes = address.get_mailboxes() + return format_html_join( + mark_safe('
    '), '{}', + [(change_url(mailbox), mailbox.name) for mailbox in boxes] + ) + display_all_mailboxes.short_description = _("Mailboxes links") + + @mark_safe + def display_forward(self, address): + forward_mailboxes = {m.name: m for m in address.get_forward_mailboxes()} + values = [] + for forward in address.forward.split(): + mbox = forward_mailboxes.get(forward) + if mbox: + values.append(admin_link()(mbox)) + else: + values.append(forward) + return '
    '.join(values) + display_forward.short_description = _("Forward") + display_forward.admin_order_field = 'forward' + + def formfield_for_dbfield(self, db_field, **kwargs): + if db_field.name == 'forward': + kwargs['widget'] = forms.TextInput(attrs={'size':'118'}) + return super(AddressAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + def get_fields(self, request, obj=None): + """ Remove mailboxes field when creating address from a popup i.e. from mailbox add form """ + fields = super(AddressAdmin, self).get_fields(request, obj) + if '_to_field' in parse_qs(request.META['QUERY_STRING']): + # Add address popup + fields = list(fields) + fields.remove('mailboxes') + return fields + + def get_queryset(self, request): + qs = super(AddressAdmin, self).get_queryset(request) + qs = qs.annotate(computed_email=Concat(F('name'), V('@'), F('domain__name'))) + return qs.annotate(Count('mailboxes')) + + def render_change_form(self, request, context, add=False, change=False, form_url='', obj=None): + if not add: + self.check_matching_mailbox(request, obj) + return super(AddressAdmin, self).render_change_form( + request, context, add, change, form_url, obj) + + def log_addition(self, request, object, *args, **kwargs): + self.check_matching_mailbox(request, object) + return super(AddressAdmin, self).log_addition(request, object, *args, **kwargs) + + def check_matching_mailbox(self, request, obj): + # Check if new addresse matches with a mbox because of having a local domain + if obj.name and obj.domain and obj.domain.name == settings.MAILBOXES_LOCAL_DOMAIN: + if obj.name not in obj.forward.split() and Mailbox.objects.filter(name=obj.name).exists(): + for mailbox in obj.mailboxes.all(): + if mailbox.name == obj.name: + return + msg = _("Address '%s' matches mailbox '%s' local address, please consider " + "if makes sense adding the mailbox on the mailboxes or forward field." + ) % (obj, obj.name) + if msg not in (m.message for m in messages.get_messages(request)): + self.message_user(request, msg, level=messages.WARNING) + + +admin.site.register(Mailbox, MailboxAdmin) +admin.site.register(Address, AddressAdmin) diff --git a/orchestra/contrib/mailboxes/api.py b/orchestra/contrib/mailboxes/api.py new file mode 100644 index 00000000..e17b68dc --- /dev/null +++ b/orchestra/contrib/mailboxes/api.py @@ -0,0 +1,28 @@ +from rest_framework import viewsets + +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import Address, Mailbox +from .serializers import AddressSerializer, MailboxSerializer, MailboxWritableSerializer + + +class AddressViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = Address.objects.select_related('domain').prefetch_related('mailboxes').all() + serializer_class = AddressSerializer + filter_fields = ('domain', 'mailboxes__name') + + +class MailboxViewSet(LogApiMixin, SetPasswordApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = Mailbox.objects.prefetch_related('addresses__domain').all() + serializer_class = MailboxSerializer + + def get_serializer_class(self): + if self.request.method == 'GET': + return self.serializer_class + + return MailboxWritableSerializer + + +router.register(r'mailboxes', MailboxViewSet) +router.register(r'addresses', AddressViewSet) diff --git a/orchestra/contrib/mailboxes/apps.py b/orchestra/contrib/mailboxes/apps.py new file mode 100644 index 00000000..395cf1e9 --- /dev/null +++ b/orchestra/contrib/mailboxes/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class MailboxesConfig(AppConfig): + name = 'orchestra.contrib.mailboxes' + verbose_name = 'Mailboxes' + + def ready(self): + from .models import Mailbox, Address + services.register(Mailbox, icon='email.png') + services.register(Address, icon='X-office-address-book.png') + from . import signals diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py new file mode 100644 index 00000000..c15c42fb --- /dev/null +++ b/orchestra/contrib/mailboxes/backends.py @@ -0,0 +1,620 @@ +import logging +import os +import re +import textwrap + +from django.core.exceptions import ObjectDoesNotExist +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor + +from . import settings +from .models import Address, Mailbox + + +logger = logging.getLogger(__name__) + + +class SieveFilteringMixin: + def generate_filter(self, mailbox, context): + name, content = mailbox.get_filtering() + for box in re.findall(r'fileinto\s+"([^"]+)"', content): + # create mailboxes if fileinfo is provided witout ':create' option + context['box'] = box + self.append(textwrap.dedent(""" + # Create %(box)s mailbox + su - %(user)s --shell /bin/bash << 'EOF' + mkdir -p "%(maildir)s/.%(box)s" + EOF + if ! grep '%(box)s' %(maildir)s/subscriptions > /dev/null; then + echo '%(box)s' >> %(maildir)s/subscriptions + chown %(user)s:%(user)s %(maildir)s/subscriptions + fi + """) % context + ) + context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context + context['filtering_cpath'] = re.sub(r'\.sieve$', '.svbin', context['filtering_path']) + if content: + context['filtering'] = ('# %(banner)s\n' + content) % context + self.append(textwrap.dedent("""\ + # Create and compile orchestra sieve filtering + su - %(user)s --shell /bin/bash << 'EOF' + mkdir -p $(dirname "%(filtering_path)s") + cat << ' EOF' > %(filtering_path)s + %(filtering)s + EOF + sievec %(filtering_path)s + EOF + """) % context + ) + else: + self.append("echo '' > %(filtering_path)s" % context) + self.append('chown %(user)s:%(group)s %(filtering_path)s' % context) + + +class UNIXUserMaildirController(SieveFilteringMixin, ServiceController): + """ + Assumes that all system users on this servers all mail accounts. + If you want to have system users AND mailboxes on the same server you should consider using virtual mailboxes. + Supports quota allocation via resources.disk.allocated. + """ + SHELL = '/dev/null' + + verbose_name = _("UNIX maildir user") + model = 'mailboxes.Mailbox' + + def save(self, mailbox): + context = self.get_context(mailbox) + self.append(textwrap.dedent(""" + # Update/create %(user)s user state + if id %(user)s ; then + old_password=$(getent shadow %(user)s | cut -d':' -f2) + usermod %(user)s \\ + --shell %(initial_shell)s \\ + --password '%(password)s' + if [[ "$old_password" != '%(password)s' ]]; then + # Postfix SASL caches passwords + RESTART_POSTFIX=1 + fi + else + useradd %(user)s \\ + --home %(home)s \\ + --password '%(password)s' + fi + mkdir -p %(home)s + chmod 751 %(home)s + chown %(user)s:%(group)s %(home)s""") % context + ) + if hasattr(mailbox, 'resources') and hasattr(mailbox.resources, 'disk'): + self.set_quota(mailbox, context) + self.generate_filter(mailbox, context) + + def set_quota(self, mailbox, context): + allocated = mailbox.resources.disk.allocated + scale = mailbox.resources.disk.resource.get_scale() + context['quota'] = allocated * scale + #unit_to_bytes(mailbox.resources.disk.unit) + self.append(textwrap.dedent(""" + # Set Maildir quota for %(user)s + su - %(user)s --shell /bin/bash << 'EOF' + mkdir -p %(maildir)s + EOF + if [ ! -f %(maildir)s/maildirsize ]; then + echo "%(quota)iS" > %(maildir)s/maildirsize + chown %(user)s:%(group)s %(maildir)s/maildirsize + else + sed -i '1s/.*/%(quota)iS/' %(maildir)s/maildirsize + fi""") % context + ) + + def delete(self, mailbox): + context = self.get_context(mailbox) + if context['deleted_home']: + self.append(textwrap.dedent("""\ + # Move home into MAILBOXES_MOVE_ON_DELETE_PATH, nesting if exists. + deleted_home="%(deleted_home)s" + while [[ -e $deleted_home ]]; do + deleted_home="${deleted_home}/$(basename ${deleted_home})" + done + mv %(home)s $deleted_home || exit_code=$? + """) % context + ) + else: + self.append("rm -fr -- %(base_home)s" % context) + self.append(textwrap.dedent(""" + nohup bash -c '{ sleep 2 && killall -u %(user)s -s KILL; }' &> /dev/null & + killall -u %(user)s || true + # Restart because of Postfix SASL caching credentials + userdel %(user)s && RESTART_POSTFIX=1 || true + groupdel %(user)s || true""") % context + ) + + def commit(self): + self.append('[[ $RESTART_POSTFIX -eq 1 ]] && service postfix restart') + super().commit() + + def get_context(self, mailbox): + context = { + 'user': mailbox.name, + 'group': mailbox.name, + 'name': mailbox.name, + 'password': mailbox.password if mailbox.active else '*%s' % mailbox.password, + 'home': mailbox.get_home(), + 'maildir': os.path.join(mailbox.get_home(), 'Maildir'), + 'initial_shell': self.SHELL, + 'banner': self.get_banner(), + } + context['deleted_home'] = settings.MAILBOXES_MOVE_ON_DELETE_PATH % context + return context + + +#class DovecotPostfixPasswdVirtualUserController(SieveFilteringMixin, ServiceController): +# """ +# WARNING: This backends is not fully implemented +# """ +# DEFAULT_GROUP = 'postfix' +# +# verbose_name = _("Dovecot-Postfix virtualuser") +# model = 'mailboxes.Mailbox' +# +# def set_user(self, context): +# self.append(textwrap.dedent(""" +# if grep '^%(user)s:' %(passwd_path)s > /dev/null ; then +# sed -i 's#^%(user)s:.*#%(passwd)s#' %(passwd_path)s +# else +# echo '%(passwd)s' >> %(passwd_path)s +# fi""") % context +# ) +# self.append("mkdir -p %(home)s" % context) +# self.append("chown %(uid)s:%(gid)s %(home)s" % context) +# +# def set_mailbox(self, context): +# self.append(textwrap.dedent(""" +# if ! grep '^%(user)s@%(mailbox_domain)s\s' %(virtual_mailbox_maps)s > /dev/null; then +# echo "%(user)s@%(mailbox_domain)s\tOK" >> %(virtual_mailbox_maps)s +# UPDATED_VIRTUAL_MAILBOX_MAPS=1 +# fi""") % context +# ) +# +# def save(self, mailbox): +# context = self.get_context(mailbox) +# self.set_user(context) +# self.set_mailbox(context) +# self.generate_filter(mailbox, context) +# +# def delete(self, mailbox): +# context = self.get_context(mailbox) +# self.append(textwrap.dedent(""" +# nohup bash -c 'sleep 2 && killall -u %(uid)s -s KILL' &> /dev/null & +# killall -u %(uid)s || true +# sed -i '/^%(user)s:.*/d' %(passwd_path)s +# sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s +# UPDATED_VIRTUAL_MAILBOX_MAPS=1""") % context +# ) +# if context['deleted_home']: +# self.append("mv %(home)s %(deleted_home)s || exit_code=$?" % context) +# else: +# self.append("rm -fr -- %(home)s" % context) +# +# def get_extra_fields(self, mailbox, context): +# context['quota'] = self.get_quota(mailbox) +# return 'userdb_mail=maildir:~/Maildir {quota}'.format(**context) +# +# def get_quota(self, mailbox): +# try: +# quota = mailbox.resources.disk.allocated +# except (AttributeError, ObjectDoesNotExist): +# return '' +# unit = mailbox.resources.disk.unit[0].upper() +# return 'userdb_quota_rule=*:bytes=%i%s' % (quota, unit) +# +# def commit(self): +# context = { +# 'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH +# } +# self.append(textwrap.dedent(""" +# [[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && { +# postmap %(virtual_mailbox_maps)s +# }""") % context +# ) +# +# def get_context(self, mailbox): +# context = { +# 'name': mailbox.name, +# 'user': mailbox.name, +# 'password': mailbox.password if mailbox.active else '*%s' % mailbox.password, +# 'uid': 10000 + mailbox.pk, +# 'gid': 10000 + mailbox.pk, +# 'group': self.DEFAULT_GROUP, +# 'quota': self.get_quota(mailbox), +# 'passwd_path': settings.MAILBOXES_PASSWD_PATH, +# 'home': mailbox.get_home(), +# 'banner': self.get_banner(), +# 'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH, +# 'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN, +# } +# context['extra_fields'] = self.get_extra_fields(mailbox, context) +# context.update({ +# 'passwd': '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context), +# 'deleted_home': settings.MAILBOXES_MOVE_ON_DELETE_PATH % context, +# }) +# return context + + +class PostfixAddressVirtualDomainController(ServiceController): + """ + Secondary SMTP server without mailboxes in it, only syncs virtual domains. + """ + verbose_name = _("Postfix address virtdomain-only") + model = 'mailboxes.Address' + related_models = ( + ('mailboxes.Mailbox', 'addresses'), + ) + doc_settings = (settings, + ('MAILBOXES_LOCAL_DOMAIN', 'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH') + ) + + def is_hosted_domain(self, domain): + """ whether or not domain MX points to this server """ + return domain.has_default_mx() + + def include_virtual_alias_domain(self, context): + domain = context['domain'] + if domain.name != context['local_domain'] and self.is_hosted_domain(domain): + self.append(textwrap.dedent(""" + # %(domain)s is a virtual domain belonging to this server + if ! grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s > /dev/null; then + echo '%(domain)s' >> %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + fi""") % context + ) + + def is_last_domain(self, domain): + return not Address.objects.filter(domain=domain).exists() + + def exclude_virtual_alias_domain(self, context): + domain = context['domain'] + if self.is_last_domain(domain): + # Prevent deleting the same domain multiple times on bulk deletes + if not hasattr(self, '_excluded_domains'): + self._excluded_domains = set() + if domain.name not in self._excluded_domains: + self._excluded_domains.add(domain.name) + self.append(textwrap.dedent(""" + # Delete %(domain)s virtual domain + if grep '^%(domain)s\s*$' %(virtual_alias_domains)s > /dev/null; then + sed -i '/^%(domain)s\s*/d' %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + fi""") % context + ) + + def save(self, address): + context = self.get_context(address) + self.include_virtual_alias_domain(context) + return context + + def delete(self, address): + context = self.get_context(address) + self.exclude_virtual_alias_domain(context) + return context + + def commit(self): + context = self.get_context_files() + self.append(textwrap.dedent(""" + [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { + service postfix reload + } + exit $exit_code + """) % context + ) + + def get_context_files(self): + return { + 'virtual_alias_domains': settings.MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH, + 'virtual_alias_maps': settings.MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH + } + + def get_context(self, address): + context = self.get_context_files() + context.update({ + 'name': address.name, + 'domain': address.domain, + 'email': address.email, + 'local_domain': settings.MAILBOXES_LOCAL_DOMAIN, + }) + return context + + +class PostfixAddressController(PostfixAddressVirtualDomainController): + """ + Addresses based on Postfix virtual alias domains, includes PostfixAddressVirtualDomainController. + """ + verbose_name = _("Postfix address") + doc_settings = (settings, ( + 'MAILBOXES_LOCAL_DOMAIN', + 'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH', + 'MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH' + )) + + def is_implicit_entry(self, context): + """ + check if virtual_alias_map entry can be omitted because the address is + equivalent to its local mbox + """ + return bool( + context['domain'].name == context['local_domain'] and + context['destination'] == context['name'] and + Mailbox.objects.filter(name=context['name']).exists()) + + def update_virtual_alias_maps(self, address, context): + context['destination'] = address.destination + if not self.is_implicit_entry(context): + self.append(textwrap.dedent(""" + # Set virtual alias entry for %(email)s + LINE='%(email)s\t%(destination)s' + if ! grep '^%(email)s\s' %(virtual_alias_maps)s > /dev/null; then + # Add new line + echo "${LINE}" >> %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 + else + # Update existing line, if needed + if ! grep "^${LINE}$" %(virtual_alias_maps)s > /dev/null; then + sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 + fi + fi""") % context) + else: + if not context['destination']: + msg = "Address %i is empty" % address.pk + self.append("\necho 'msg' >&2" % msg) + logger.warning(msg) + else: + self.append("\n# %(email)s %(destination)s entry is redundant" % context) + self.exclude_virtual_alias_maps(context) + # Virtual mailbox stuff +# destination = [] +# for mailbox in address.get_mailboxes(): +# context['mailbox'] = mailbox +# destination.append("%(mailbox)s@%(local_domain)s" % context) +# for forward in address.forward: +# if '@' in forward: +# destination.append(forward) + + def exclude_virtual_alias_maps(self, context): + self.append(textwrap.dedent("""\ + # Remove %(email)s virtual alias entry + if grep '^%(email)s\s' %(virtual_alias_maps)s > /dev/null; then + sed -i '/^%(email)s\s/d' %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 + fi""") % context + ) + + def save(self, address): + context = super().save(address) + self.update_virtual_alias_maps(address, context) + + def delete(self, address): + context = super().delete(address) + self.exclude_virtual_alias_maps(context) + + def commit(self): + context = self.get_context_files() + self.append(textwrap.dedent(""" + # Apply changes if needed + [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { + service postfix reload + } + [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { + postmap %(virtual_alias_maps)s + } + exit $exit_code + """) % context + ) + + +class AutoresponseController(ServiceController): + """ + WARNING: not implemented + """ + verbose_name = _("Mail autoresponse") + model = 'mailboxes.Autoresponse' + + +class DovecotMaildirDisk(ServiceMonitor): + """ + Maildir disk usage based on Dovecot maildirsize file + http://wiki2.dovecot.org/Quota/Maildir + """ + model = 'mailboxes.Mailbox' + resource = ServiceMonitor.DISK + verbose_name = _("Dovecot Maildir size") + delete_old_equal_values = True + doc_settings = (settings, + ('MAILBOXES_MAILDIRSIZE_PATH',) + ) + + def prepare(self): + super().prepare() + current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") + # self.append(textwrap.dedent("""\ + # function monitor () { + # awk 'BEGIN { size = 0 } NR > 1 { size += $1 } END { print size }' $1 || echo 0 + # }""")) + self.append(textwrap.dedent("""\ + function monitor () { + SIZE=$(du -sb $1/Maildir/ 2> /dev/null || echo 0) && echo $SIZE | awk '{print $1}' + }""")) + + def monitor(self, mailbox): + context = self.get_context(mailbox) + # self.append("echo %(object_id)s $(monitor %(maildir_path)s)" % context) + self.append("echo %(object_id)s $(monitor %(home)s)" % context) + + def get_context(self, mailbox): + context = { + 'home': mailbox.get_home(), + 'object_id': mailbox.pk + } + context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context + return context + + +class PostfixMailscannerTraffic(ServiceMonitor): + """ + A high-performance log parser. + Reads the mail.log file only once, for all users. + """ + model = 'mailboxes.Mailbox' + resource = ServiceMonitor.TRAFFIC + verbose_name = _("Postfix-Mailscanner traffic") + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + doc_settings = (settings, + ('MAILBOXES_MAIL_LOG_PATH',) + ) + + def prepare(self): + mail_log = settings.MAILBOXES_MAIL_LOG_PATH + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'mail_logs': str((mail_log, mail_log+'.1')), + } + self.append(textwrap.dedent("""\ + import re + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + # Converts orchestra's UTC dates to local timezone + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + maillogs = {mail_logs} + end_datetime = to_local_timezone('{current_date}') + end_date = int(end_datetime.strftime('%Y%m%d%H%M%S')) + months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') + months = dict((m, '%02d' % n) for n, m in enumerate(months, 1)) + + def inside_period(month, day, time, ini_date): + global months + global end_datetime + # Mar 9 17:13:22 + month = months[month] + year = end_datetime.year + if month == '12' and end_datetime.month == 1: + year = year+1 + if len(day) == 1: + day = '0' + day + date = str(year) + month + day + date += time.replace(':', '') + return ini_date < int(date) < end_date + + users = {{}} + delivers = {{}} + reverse = {{}} + + def prepare(object_id, mailbox, ini_date): + global users + global delivers + global reverse + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + users[mailbox] = (ini_date, object_id) + delivers[mailbox] = set() + reverse[mailbox] = set() + + def monitor(users, delivers, reverse, maillogs): + targets = {{}} + counter = {{}} + user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)') + for maillog in maillogs: + try: + with open(maillog, 'r') as maillog: + for line in maillog.readlines(): + # Only search for Authenticated sendings + if '(Authenticated sender: ' in line: + username = user_regex.search(line).groups()[0] + try: + sender = users[username] + except KeyError: + continue + else: + month, day, time, __, proc, id = line.split()[:6] + if inside_period(month, day, time, sender[0]): + # Add new email + delivers[id[:-1]] = username + # Look for a MailScanner requeue ID + elif ' Requeue: ' in line: + id, __, req_id = line.split()[6:9] + id = id.split('.')[0] + try: + username = delivers[id] + except KeyError: + pass + else: + targets[req_id] = (username, 0) + reverse[username].add(req_id) + # Look for the mail size and count the number of recipients of each email + else: + try: + month, day, time, __, proc, req_id, __, msize = line.split()[:8] + except ValueError: + # not interested in this line + continue + if proc.startswith('postfix/'): + req_id = req_id[:-1] + if msize.startswith('size='): + try: + target = targets[req_id] + except KeyError: + pass + else: + targets[req_id] = (target[0], int(msize[5:-1])) + elif proc.startswith('postfix/smtp'): + try: + target = targets[req_id] + except KeyError: + pass + else: + if inside_period(month, day, time, users[target[0]][0]): + try: + counter[req_id] += 1 + except KeyError: + counter[req_id] = 1 + except IOError as e: + sys.stderr.write(str(e)+'\\n') + + for username, opts in users.iteritems(): + size = 0 + for req_id in reverse[username]: + size += targets[req_id][1] * counter.get(req_id, 0) + print opts[1], size + """).format(**context) + ) + + def commit(self): + self.append('monitor(users, delivers, reverse, maillogs)') + + def monitor(self, mailbox): + context = self.get_context(mailbox) + self.append("prepare(%(object_id)s, '%(mailbox)s', '%(last_date)s')" % context) + + def get_context(self, mailbox): + context = { + 'mailbox': mailbox.name, + 'object_id': mailbox.pk, + 'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + return context + +class RoundcubeIdentityController(ServiceController): + """ + WARNING: not implemented + """ + verbose_name = _("Roundcube Identity Controller") + model = 'mailboxes.Mailbox' + diff --git a/orchestra/contrib/mailboxes/filters.py b/orchestra/contrib/mailboxes/filters.py new file mode 100644 index 00000000..2c1dd603 --- /dev/null +++ b/orchestra/contrib/mailboxes/filters.py @@ -0,0 +1,47 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + + +class HasMailboxListFilter(SimpleListFilter): + """ Filter addresses whether they have any mailbox or not """ + title = _("has mailbox") + parameter_name = 'has_mailbox' + + def lookups(self, request, model_admin): + return ( + ('True', _("True")), + ('False', _("False")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(mailboxes__isnull=False) + elif self.value() == 'False': + return queryset.filter(mailboxes__isnull=True) + return queryset + + +class HasForwardListFilter(HasMailboxListFilter): + """ Filter addresses whether they have any mailbox or not """ + title = _("has forward") + parameter_name = 'has_forward' + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.exclude(forward='') + elif self.value() == 'False': + return queryset.filter(forward='') + return queryset + + +class HasAddressListFilter(HasMailboxListFilter): + """ Filter addresses whether they have any mailbox or not """ + title = _("has address") + parameter_name = 'has_address' + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(addresses__isnull=False) + elif self.value() == 'False': + return queryset.filter(addresses__isnull=True) + return queryset diff --git a/orchestra/contrib/mailboxes/forms.py b/orchestra/contrib/mailboxes/forms.py new file mode 100644 index 00000000..522b608e --- /dev/null +++ b/orchestra/contrib/mailboxes/forms.py @@ -0,0 +1,79 @@ +from django import forms +from django.contrib.admin import widgets +from django.core.exceptions import ValidationError +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.forms import UserCreationForm, UserChangeForm +from orchestra.utils.python import AttrDict + +from . import settings +from .models import Address, Mailbox + + +class MailboxForm(forms.ModelForm): + """ hacky form for adding reverse M2M form field for Mailbox.addresses """ + # TODO keep track of this ticket for future reimplementation + # https://code.djangoproject.com/ticket/897 + addresses = forms.ModelMultipleChoiceField(required=False, + queryset=Address.objects.select_related('domain'), + widget=widgets.FilteredSelectMultiple(verbose_name=_('addresses'), is_stacked=False)) + + def __init__(self, *args, **kwargs): + super(MailboxForm, self).__init__(*args, **kwargs) + # Hack the widget in order to display add button + remote_field_mock = AttrDict(**{ + 'model': Address, + 'get_related_field': lambda: AttrDict(name='id'), + + }) + widget = self.fields['addresses'].widget + self.fields['addresses'].widget = widgets.RelatedFieldWidgetWrapper( + widget, remote_field_mock, self.modeladmin.admin_site, can_add_related=True) + + account = self.modeladmin.account + # Filter related addresses by account + old_render = self.fields['addresses'].widget.render + def render(*args, **kwargs): + output = old_render(*args, **kwargs) + args = 'account=%i&mailboxes=%s' % (account.pk, self.instance.pk) + output = output.replace('/add/?', '/add/?%s&' % args) + return mark_safe(output) + self.fields['addresses'].widget.render = render + queryset = self.fields['addresses'].queryset + realted_addresses = queryset.filter(account_id=account.pk).order_by('name') + self.fields['addresses'].queryset = realted_addresses + + if self.instance and self.instance.pk: + self.fields['addresses'].initial = self.instance.addresses.all() + + def clean_name(self): + name = self.cleaned_data['name'] + max_length = settings.MAILBOXES_NAME_MAX_LENGTH + if len(name) > max_length: + raise ValidationError("Name length should be less than %i." % max_length) + return name + + +class MailboxChangeForm(UserChangeForm, MailboxForm): + pass + + +class MailboxCreationForm(UserCreationForm, MailboxForm): + def clean_name(self): + # Since model.clean() will check this, this is redundant, + # but it sets a nicer error message than the ORM and avoids conflicts with contrib.auth + name = super().clean_name() + try: + self._meta.model._default_manager.get(name=name) + except self._meta.model.DoesNotExist: + return name + raise forms.ValidationError(self.error_messages['duplicate_username']) + + +class AddressForm(forms.ModelForm): + def clean(self): + cleaned_data = super(AddressForm, self).clean() + forward = cleaned_data.get('forward', '') + if not cleaned_data.get('mailboxes', True) and not forward: + raise ValidationError(_("Mailboxes or forward address should be provided.")) diff --git a/orchestra/contrib/mailboxes/models.py b/orchestra/contrib/mailboxes/models.py new file mode 100644 index 00000000..7122b5e1 --- /dev/null +++ b/orchestra/contrib/mailboxes/models.py @@ -0,0 +1,178 @@ +import os +import re +from collections import defaultdict + +from django.contrib.auth.hashers import make_password +from django.core.validators import RegexValidator, ValidationError +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from . import validators, settings + + +class Mailbox(models.Model): + CUSTOM = 'CUSTOM' + + name = models.CharField(_("name"), unique=True, db_index=True, + max_length=settings.MAILBOXES_NAME_MAX_LENGTH, + help_text=_("Required. %s characters or fewer. Letters, digits and ./-/_ only.") % + settings.MAILBOXES_NAME_MAX_LENGTH, + validators=[ + RegexValidator(r'^[\w.-]+$', _("Enter a valid mailbox name.")), + ]) + password = models.CharField(_("password"), max_length=128) + account = models.ForeignKey('accounts.Account', verbose_name=_("account"), + related_name='mailboxes', on_delete=models.CASCADE) + filtering = models.CharField(max_length=16, + default=settings.MAILBOXES_MAILBOX_DEFAULT_FILTERING, + choices=[(k, v[0]) for k,v in sorted(settings.MAILBOXES_MAILBOX_FILTERINGS.items())]) + custom_filtering = models.TextField(_("filtering"), blank=True, + validators=[validators.validate_sieve], + help_text=_("Arbitrary email filtering in " + "sieve language. " + "This overrides any automatic junk email filtering")) + is_active = models.BooleanField(_("active"), default=True) + + class Meta: + verbose_name_plural = _("mailboxes") + + def __str__(self): + return self.name + + @cached_property + def active(self): + try: + return self.is_active and self.account.is_active + except type(self).account.field.related_model.DoesNotExist: + return self.is_active + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def set_password(self, raw_password): + self.password = make_password(raw_password) + + def get_home(self): + context = { + 'name': self.name, + 'username': self.name, + } + return os.path.normpath(settings.MAILBOXES_HOME % context) + + def clean(self): + if self.filtering == self.CUSTOM and not self.custom_filtering: + raise ValidationError({ + 'custom_filtering': _("Custom filtering is selected but not provided.") + }) + + def get_filtering(self): + name, content = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering] + if callable(content): + # Custom filtering + content = content(self) + return (name, content) + + def get_local_address(self): + if not settings.MAILBOXES_LOCAL_DOMAIN: + raise AttributeError("Mailboxes do not have a defined local address domain.") + return '@'.join((self.name, settings.MAILBOXES_LOCAL_DOMAIN)) + + def get_forwards(self): + return Address.objects.filter(forward__regex=r'(^|.*\s)%s(\s.*|$)' % self.name) + + def get_addresses(self): + mboxes = self.addresses.all() + forwards = self.get_forwards() + return set(mboxes).union(set(forwards)) + + +class Address(models.Model): + name = models.CharField(_("name"), max_length=64, blank=True, + validators=[validators.validate_emailname], + help_text=_("Address name, left blank for a catch-all address")) + domain = models.ForeignKey(settings.MAILBOXES_DOMAIN_MODEL, + verbose_name=_("domain"), related_name='addresses', on_delete=models.CASCADE) + mailboxes = models.ManyToManyField(Mailbox, verbose_name=_("mailboxes"), + related_name='addresses', blank=True) + forward = models.CharField(_("forward"), max_length=256, blank=True, + validators=[validators.validate_forward], + help_text=_("Space separated email addresses or mailboxes")) + account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), + related_name='addresses', on_delete=models.CASCADE) + + class Meta: + verbose_name_plural = _("addresses") + unique_together = ('name', 'domain') + + def __str__(self): + return self.email + + @property + def email(self): + return "%s@%s" % (self.name, self.domain) + + @cached_property + def destination(self): + destinations = list(self.mailboxes.values_list('name', flat=True)) + if self.forward: + destinations += self.forward.split() + return ' '.join(destinations) + + def clean(self): + errors = defaultdict(list) + local_domain = settings.MAILBOXES_LOCAL_DOMAIN + if local_domain: + forwards = self.forward.split() + for ix, forward in enumerate(forwards): + if forward.endswith('@%s' % local_domain): + name = forward.split('@')[0] + if Mailbox.objects.filter(name=name).exists(): + forwards[ix] = name + self.forward = ' '.join(forwards) + if self.account_id: + for mailbox in self.get_forward_mailboxes(): + if mailbox.account_id == self.account_id: + errors['forward'].append( + _("Please use mailboxes field for '%s' mailbox.") % mailbox + ) + if self.domain: + for forward in self.forward.split(): + if self.email == forward: + errors['forward'].append( + _("'%s' forwards to itself.") % forward + ) + if errors: + raise ValidationError(errors) + + def get_forward_mailboxes(self): + rm_local_domain = re.compile(r'@%s$' % settings.MAILBOXES_LOCAL_DOMAIN) + mailboxes = [] + for forward in self.forward.split(): + forward = rm_local_domain.sub('', forward) + if '@' not in forward: + mailboxes.append(forward) + return Mailbox.objects.filter(name__in=mailboxes) + + def get_mailboxes(self): + for mailbox in self.mailboxes.all(): + yield mailbox + for mailbox in self.get_forward_mailboxes(): + yield mailbox + + +class Autoresponse(models.Model): + address = models.OneToOneField(Address, verbose_name=_("address"), + related_name='autoresponse', on_delete=models.CASCADE) + # TODO initial_date + subject = models.CharField(_("subject"), max_length=256) + message = models.TextField(_("message")) + enabled = models.BooleanField(_("enabled"), default=False) + + def __str__(self): + return self.address diff --git a/orchestra/contrib/mailboxes/serializers.py b/orchestra/contrib/mailboxes/serializers.py new file mode 100644 index 00000000..1608a6ce --- /dev/null +++ b/orchestra/contrib/mailboxes/serializers.py @@ -0,0 +1,107 @@ +from django.db import transaction +from rest_framework import serializers + +from orchestra.api.serializers import SetPasswordHyperlinkedSerializer, RelatedHyperlinkedModelSerializer +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .models import Mailbox, Address + + +class RelatedDomainSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = Address.domain.field.related_model + fields = ('url', 'id', 'name') + + +class RelatedAddressSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + domain = RelatedDomainSerializer() + + class Meta: + model = Address + fields = ('url', 'id', 'name', 'domain', 'forward') +# +# def from_native(self, data, files=None): +# queryset = self.opts.model.objects.filter(account=self.account) +# return get_object_or_404(queryset, name=data['name']) + + +class MailboxSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): + addresses = RelatedAddressSerializer(many=True, read_only=True) + + class Meta: + model = Mailbox + fields = ( + 'url', 'id', 'name', 'password', 'filtering', 'custom_filtering', 'addresses', 'is_active' + ) + postonly_fields = ('name', 'password') + + +class AddressRelatedField(serializers.HyperlinkedRelatedField): + # Filter addresses by account (user) + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(account=self.context['account']) + + +class MailboxWritableSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): + addresses = AddressRelatedField(many=True, view_name='address-detail', queryset=Address.objects.all()) + + class Meta: + model = Mailbox + fields = ( + 'url', 'id', 'name', 'password', 'filtering', 'custom_filtering', 'addresses', 'is_active' + ) + postonly_fields = ('name', 'password') + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['addresses'].context['account'] = self.account + + @transaction.atomic + def create(self, validated_data): + addresses = validated_data.pop('addresses', []) + instance = super().create(validated_data) + instance.addresses.set(addresses) + return instance + + @transaction.atomic + def update(self, instance, validated_data): + addresses = validated_data.pop('addresses', []) + instance.addresses.set(addresses) + return super().update(instance, validated_data) + + +class RelatedMailboxSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = Mailbox + fields = ('url', 'id', 'name') + + +class AddressSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + domain = RelatedDomainSerializer() + mailboxes = RelatedMailboxSerializer(many=True, required=False) + + class Meta: + model = Address + fields = ('url', 'id', 'name', 'domain', 'mailboxes', 'forward') + + def validate(self, attrs): + attrs = super(AddressSerializer, self).validate(attrs) + mailboxes = attrs.get('mailboxes', []) + forward = attrs.get('forward', '') + if not mailboxes and not forward: + raise serializers.ValidationError("A mailbox or forward address should be provided.") + return attrs + + @transaction.atomic + def create(self, validated_data): + mailboxes = validated_data.pop('mailboxes', []) + obj = super().create(validated_data) + obj.mailboxes.set(mailboxes) + return obj + + @transaction.atomic + def update(self, instance, validated_data): + mailboxes = validated_data.pop('mailboxes', []) + instance.mailboxes.set(mailboxes) + return super().update(instance, validated_data) diff --git a/orchestra/contrib/mailboxes/settings.py b/orchestra/contrib/mailboxes/settings.py new file mode 100644 index 00000000..c9412754 --- /dev/null +++ b/orchestra/contrib/mailboxes/settings.py @@ -0,0 +1,205 @@ +import os +import textwrap + +from django.utils.functional import lazy +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting +from orchestra.core.validators import validate_name +from orchestra.settings import ORCHESTRA_BASE_DOMAIN + + +_names = ('name', 'username',) +_backend_names = _names + ('user', 'group', 'home') +mark_safe_lazy = lazy(mark_safe, str) + + +MAILBOXES_DOMAIN_MODEL = Setting('MAILBOXES_DOMAIN_MODEL', 'domains.Domain', + validators=[Setting.validate_model_label] +) + + +MAILBOXES_NAME_MAX_LENGTH = Setting('MAILBOXES_NAME_MAX_LENGTH', + 32, + help_text=_("Limit for system user based mailbox on Linux is 32.") +) + + +MAILBOXES_HOME = Setting('MAILBOXES_HOME', + '/home/%(name)s', + help_text="Available fromat names: %s" % ', '.join(_names), + validators=[Setting.string_format_validator(_names)], +) + + +MAILBOXES_SIEVE_PATH = Setting('MAILBOXES_SIEVE_PATH', + os.path.join('%(home)s/sieve/orchestra.sieve'), + help_text="If you are using Dovecot you can use " + "" + "sieve_before in order to make sure orchestra sieve script is exectued." + "
    Available fromat names: %s" % ', '.join(_names), + validators=[Setting.string_format_validator(_backend_names)], +) + + +MAILBOXES_SIEVETEST_PATH = Setting('MAILBOXES_SIEVETEST_PATH', + '/dev/shm' +) + + +MAILBOXES_SIEVETEST_BIN_PATH = Setting('MAILBOXES_SIEVETEST_BIN_PATH', + '%(orchestra_root)s/bin/sieve-test', + validators=[Setting.string_format_validator(('orchestra_root',))] +) + + +MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH = Setting('MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH', + '/etc/postfix/virtual_mailboxes' +) + + +MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH = Setting('MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH', + '/etc/postfix/virtual_aliases' +) + + +MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH = Setting('MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH', + '/etc/postfix/virtual_domains' +) + + +MAILBOXES_LOCAL_DOMAIN = Setting('MAILBOXES_LOCAL_DOMAIN', + ORCHESTRA_BASE_DOMAIN, + validators=[validate_name], + help_text="Defaults to ORCHESTRA_BASE_DOMAIN." +) + + +MAILBOXES_PASSWD_PATH = Setting('MAILBOXES_PASSWD_PATH', + '/etc/dovecot/passwd' +) + + +MAILBOXES_SPAM_SCORE_HEADER = Setting('MAILBOXES_SPAM_SCORE_HEADER', + 'X-Spam-Score' +) + + +MAILBOXES_SPAM_SCORE_SYMBOL = Setting('MAILBOXES_SPAM_SCORE_SYMBOL', + '', + help_text="Blank for numeric spam score.", +) + + +MAILBOXES_MAILBOX_FILTERINGS = Setting('MAILBOXES_MAILBOX_FILTERINGS', + { + # value: (verbose_name, filter) + 'DISABLE': (_("Disable"), ''), + 'REJECT': (mark_safe_lazy(_("Reject spam (Score≥8)")), ( + textwrap.dedent("""\ + if header :contains "%(score_header)s" "%(score_value)s" { + discard; + stop; + }""") if MAILBOXES_SPAM_SCORE_SYMBOL else + textwrap.dedent("""\ + require ["relational","comparator-i;ascii-numeric"]; + if allof ( + not header :matches "%(score_header)s" "-*", + header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "8" ) + { + discard; + stop; + }""")) % { + 'score_header': MAILBOXES_SPAM_SCORE_HEADER, + 'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*8 + } + ), + 'REJECT5': (mark_safe_lazy(_("Reject spam (Score≥5)")), ( + textwrap.dedent("""\ + if header :contains "%(score_header)s" "%(score_value)s" { + discard; + stop; + }""") if MAILBOXES_SPAM_SCORE_SYMBOL else + textwrap.dedent("""\ + require ["relational","comparator-i;ascii-numeric"]; + if allof ( + not header :matches "%(score_header)s" "-*", + header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "5" ) + { + discard; + stop; + }""")) % { + 'score_header': MAILBOXES_SPAM_SCORE_HEADER, + 'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*5 + } + ), + 'REDIRECT': (mark_safe_lazy(_("Archive spam (Score≥8)")), ( + textwrap.dedent("""\ + require "fileinto"; + if header :contains "%(score_header)s" "%(score_value)s" { + fileinto "Spam"; + stop; + }""") if MAILBOXES_SPAM_SCORE_SYMBOL else + textwrap.dedent("""\ + require ["fileinto","relational","comparator-i;ascii-numeric"]; + if allof ( + not header :matches "%(score_header)s" "-*", + header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "8" ) + { + fileinto "Spam"; + stop; + }""")) % { + 'score_header': MAILBOXES_SPAM_SCORE_HEADER, + 'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*8 + } + ), + 'REDIRECT5': (mark_safe_lazy(_("Archive spam (Score≥5)")), ( + textwrap.dedent("""\ + require "fileinto"; + if header :contains "%(score_header)s" "%(score_value)s" { + fileinto "Spam"; + stop; + }""") if MAILBOXES_SPAM_SCORE_SYMBOL else + textwrap.dedent("""\ + require ["fileinto","relational","comparator-i;ascii-numeric"]; + if allof ( + not header :matches "%(score_header)s" "-*", + header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "5" ) + { + fileinto "Spam"; + stop; + }""")) % { + 'score_header': MAILBOXES_SPAM_SCORE_HEADER, + 'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*5 + } + ), + 'CUSTOM': (_("Custom filtering"), lambda mailbox: mailbox.custom_filtering), + } +) + + +MAILBOXES_MAILBOX_DEFAULT_FILTERING = Setting('MAILBOXES_MAILBOX_DEFAULT_FILTERING', + 'REDIRECT', + choices=tuple((k, v[0]) for k,v in MAILBOXES_MAILBOX_FILTERINGS.items()) +) + + +MAILBOXES_MAILDIRSIZE_PATH = Setting('MAILBOXES_MAILDIRSIZE_PATH', + '%(home)s/Maildir/maildirsize', + help_text="Available fromat names: %s" % ', '.join(_backend_names), + validators=[Setting.string_format_validator(_backend_names)], +) + + + +MAILBOXES_MAIL_LOG_PATH = Setting('MAILBOXES_MAIL_LOG_PATH', + '/var/log/mail.log' +) + + +MAILBOXES_MOVE_ON_DELETE_PATH = Setting('MAILBOXES_MOVE_ON_DELETE_PATH', + '', + help_text="Available fromat names: %s" % ', '.join(_backend_names), + validators=[Setting.string_format_validator(_backend_names)], +) diff --git a/orchestra/contrib/mailboxes/signals.py b/orchestra/contrib/mailboxes/signals.py new file mode 100644 index 00000000..5cfdcef0 --- /dev/null +++ b/orchestra/contrib/mailboxes/signals.py @@ -0,0 +1,51 @@ +from django.db.models.signals import pre_save, post_delete, post_save +from django.dispatch import receiver + +from . import settings +from .models import Mailbox, Address + + +# Admin bulk deletion doesn't call model.delete() +# So, signals are used instead of model method overriding + +@receiver(post_delete, sender=Mailbox, dispatch_uid='mailboxes.delete_forwards') +def delete_forwards(sender, *args, **kwargs): + # Cleanup related addresses + instance = kwargs['instance'] + for address in instance.get_forwards(): + forward = address.forward.split() + forward.remove(instance.name) + address.forward = ' '.join(forward) + if not address.destination: + address.delete() + else: + address.save() + + +@receiver(pre_save, sender=Mailbox, dispatch_uid='mailboxes.create_local_address') +def create_local_address(sender, *args, **kwargs): + mbox = kwargs['instance'] + local_domain = settings.MAILBOXES_LOCAL_DOMAIN + if not mbox.pk and local_domain: + Domain = Address._meta.get_field('domain').remote_field.model + try: + domain = Domain.objects.get(name=local_domain) + except Domain.DoesNotExist: + pass + else: + addr, created = Address.objects.get_or_create( + name=mbox.name, domain=domain, account_id=domain.account_id) + if created: + if domain.account_id == mbox.account_id: + mbox._post_save_add_address = addr + else: + addr.forward = mbox.name + addr.save(update_fields=('forward',)) + + +@receiver(post_save, sender=Mailbox, dispatch_uid='mailboxes.add_local_address') +def add_local_address(sender, *args, **kwargs): + mbox = kwargs['instance'] + addr = getattr(mbox, '_post_save_add_address', None) + if addr: + addr.mailboxes.add(mbox) diff --git a/orchestra/contrib/mailboxes/tests/__init__.py b/orchestra/contrib/mailboxes/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/mailboxes/tests/functional_tests/__init__.py b/orchestra/contrib/mailboxes/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/mailboxes/tests/functional_tests/tests.py b/orchestra/contrib/mailboxes/tests/functional_tests/tests.py new file mode 100644 index 00000000..6ae693bb --- /dev/null +++ b/orchestra/contrib/mailboxes/tests/functional_tests/tests.py @@ -0,0 +1,380 @@ +import imaplib +import os +import poplib +import smtplib +import time +import textwrap +import unittest +from email.mime.text import MIMEText + +from django.apps import apps +from django.conf import settings as djsettings +from django.contrib.contenttypes.models import ContentType +from django.core.management.base import CommandError +from django.urls import reverse +from selenium.webdriver.support.select import Select + +from orchestra.contrib.orchestration.models import Server, Route +from orchestra.contrib.resources.models import Resource +from orchestra.utils.sys import sshrun +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error, save_response_on_error + +from ... import backends, settings +from ...models import Mailbox + +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + + +class MailboxMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') + DEPENDENCIES = ( + 'orchestra.contrib.orchestration', + 'orchestra.contrib.mails', + 'orchestra.contrib.resources', + ) + + def setUp(self): + super(MailboxMixin, self).setUp() + self.add_route() + # clean resource relation from other tests + apps.get_app_config('resources').reload_relations() + djsettings.DEBUG = True + + def add_route(self): + server = Server.objects.create(name=self.MASTER_SERVER) + backend = backends.RoundcubeIdentityController.get_name() + Route.objects.create(backend=backend, match=True, host=server) + backend = backends.PostfixAddressController.get_name() + Route.objects.create(backend=backend, match=True, host=server) + + def add_quota_resource(self): + Resource.objects.create( + name='disk', + content_type=ContentType.objects.get_for_model(Mailbox), + period=Resource.LAST, + verbose_name='Mail quota', + unit='MB', + scale=10**6, + on_demand=False, + default_allocation=2000 + ) + + def save(self): + raise NotImplementedError + + def add(self): + raise NotImplementedError + + def delete(self): + raise NotImplementedError + + def update(self): + raise NotImplementedError + + def disable(self): + raise NotImplementedError + + def add_group(self, username, groupname): + raise NotImplementedError + + def login_imap(self, username, password): + mail = imaplib.IMAP4_SSL(self.MASTER_SERVER) + status, msg = mail.login(username, password) + self.assertEqual('OK', status) + self.assertEqual(['Logged in'], msg) + return mail + + def login_pop3(self, username, password): + pop = poplib.POP3(self.MASTER_SERVER) + pop.user(username) + pop.pass_(password) + return pop + + def send_email(self, to, token): + msg = MIMEText(token) + msg['To'] = to + msg['From'] = 'orchestra@%s' % self.MASTER_SERVER + msg['Subject'] = 'test' + server = smtplib.SMTP(self.MASTER_SERVER, 25) + try: + server.ehlo() + server.starttls() + server.ehlo() + server.sendmail(msg['From'], msg['To'], msg.as_string()) + finally: + server.quit() + + def validate_mailbox(self, username): + sshrun(self.MASTER_SERVER, "doveadm search -u %s ALL" % username, display=False) + + def validate_email(self, username, token): + home = Mailbox.objects.get(name=username).get_home() + sshrun(self.MASTER_SERVER, "grep '%s' %s/Maildir/new/*" % (token, home), display=False) + + def test_add(self): + username = '%s_mailbox' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + imap = self.login_imap(username, password) + self.validate_mailbox(username) + + def test_change_password(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + imap = self.login_imap(username, password) + new_password = '@!?%spppP001' % random_ascii(5) + self.change_password(username, new_password) + imap = self.login_imap(username, new_password) + + def test_quota(self): + username = '%s_mailbox' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add_quota_resource() + quota = 100 + self.add(username, password, quota=quota) + self.addCleanup(self.delete, username) + get_quota = "doveadm quota get -u %s 2>&1|grep STORAGE|awk {'print $5'}" % username + stdout = sshrun(self.MASTER_SERVER, get_quota, display=False).stdout + self.assertEqual(quota*1024, int(stdout)) + imap = self.login_imap(username, password) + imap_quota = int(imap.getquotaroot("INBOX")[1][1][0].split(' ')[-1].split(')')[0]) + self.assertEqual(quota*1024, imap_quota) + + def test_send_email(self): + username = '%s_mailbox' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + msg = MIMEText("Hola bishuns") + msg['To'] = 'noexists@example.com' + msg['From'] = '%s@%s' % (username, self.MASTER_SERVER) + msg['Subject'] = "test" + server = smtplib.SMTP(self.MASTER_SERVER, 25) + server.login(username, password) + try: + server.sendmail(msg['From'], msg['To'], msg.as_string()) + finally: + server.quit() + + def test_address(self): + username = '%s_mailbox' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + domain = '%s_domain.lan' % random_ascii(5) + name = '%s_name' % random_ascii(5) + domain = self.account.domains.create(name=domain) + self.add_address(username, name, domain) + token = random_ascii(100) + self.send_email("%s@%s" % (name, domain), token) + self.validate_email(username, token) + + def test_disable(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.validate_mailbox(username) +# self.addCleanup(self.delete, username) + imap = self.login_imap(username, password) + self.disable(username) + self.assertRaises(imap.error, self.login_imap, username, password) + + def test_delete(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%sppppP001' % random_ascii(5) + self.add(username, password) + imap = self.login_imap(username, password) + self.validate_mailbox(username) + mailbox = Mailbox.objects.get(name=username) + home = mailbox.get_home() + self.delete(username) + self.assertRaises(Mailbox.DoesNotExist, Mailbox.objects.get, name=username) + self.assertRaises(CommandError, self.validate_mailbox, username) + self.assertRaises(imap.error, self.login_imap, username, password) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'ls %s' % home, display=False) + + def test_delete_address(self): + username = '%s_mailbox' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + domain = '%s_domain.lan' % random_ascii(5) + name = '%s_name' % random_ascii(5) + domain = self.account.domains.create(name=domain) + self.add_address(username, name, domain) + token = random_ascii(100) + self.send_email("%s@%s" % (name, domain), token) + self.validate_email(username, token) + self.delete_address(username) + self.send_email("%s@%s" % (name, domain), token) + self.validate_email(username, token) + + def test_custom_filtering(self): + username = '%s_mailbox' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + folder = random_ascii(5) + filtering = textwrap.dedent(""" + require "fileinto"; + if true { + fileinto "%s"; + stop; + }""" % folder) + self.add(username, password, filtering=filtering) + self.addCleanup(self.delete, username) + imap = self.login_imap(username, password) + imap.create(folder) + self.validate_mailbox(username) + token = random_ascii(100) + self.send_email("%s@%s" % (username, settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN), token) + home = Mailbox.objects.get(name=username).get_home() + sshrun(self.MASTER_SERVER, + "grep '%s' %s/Maildir/.%s/new/*" % (token, home, folder), display=False) + +# TODO test update shit +# TODO test autoreply + + +@unittest.skipUnless(TEST_REST_API, "REST API tests") +class RESTMailboxMixin(MailboxMixin): + def setUp(self): + super(RESTMailboxMixin, self).setUp() + self.rest_login() + + @save_response_on_error + def add(self, username, password, quota=None, filtering=None): + extra = {} + if quota: + extra.update({ + "resources": [ + { + "name": "disk", + "allocated": quota + }, + ] + }) + if filtering: + extra.update({ + 'filtering': 'CUSTOM', + 'custom_filtering': filtering, + }) + self.rest.mailboxes.create(name=username, password=password, **extra) + + @save_response_on_error + def delete(self, username): + mailbox = self.rest.mailboxes.retrieve(name=username).get() + mailbox.delete() + + @save_response_on_error + def change_password(self, username, password): + mailbox = self.rest.mailboxes.retrieve(name=username).get() + mailbox.change_password(password) + + @save_response_on_error + def add_address(self, username, name, domain): + mailbox = self.rest.mailboxes.retrieve(name=username).get() + domain = self.rest.domains.retrieve(name=domain.name).get() + self.rest.addresses.create(name=name, domain=domain, mailboxes=[mailbox]) + + @save_response_on_error + def delete_address(self, username): + mailbox = self.rest.mailboxes.retrieve(name=username).get() + self.rest.addresses.delete() + + @save_response_on_error + def disable(self, username): + mailbox = self.rest.mailboxes.retrieve(name=username).get() + mailbox.update(is_active=False) + + +class AdminMailboxMixin(MailboxMixin): + def setUp(self): + super(AdminMailboxMixin, self).setUp() + self.admin_login() + + @snapshot_on_error + def add(self, username, password, quota=None, filtering=None): + url = self.live_server_url + reverse('admin:mailboxes_mailbox_add') + self.selenium.get(url) + +# account_input = self.selenium.find_element_by_id('id_account') +# account_select = Select(account_input) +# account_select.select_by_value(str(self.account.pk)) + + name_field = self.selenium.find_element_by_id('id_name') + name_field.send_keys(username) + + password_field = self.selenium.find_element_by_id('id_password1') + password_field.send_keys(password) + password_field = self.selenium.find_element_by_id('id_password2') + password_field.send_keys(password) + + if quota is not None: + quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated' + quota_field = self.selenium.find_element_by_id(quota_id) + quota_field.clear() + quota_field.send_keys(quota) + + if filtering is not None: + filtering_input = self.selenium.find_element_by_id('id_filtering') + filtering_select = Select(filtering_input) + filtering_select.select_by_value("CUSTOM") + filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0') + filtering_inline.click() + time.sleep(0.5) + filtering_field = self.selenium.find_element_by_id('id_custom_filtering') + filtering_field.send_keys(filtering) + + name_field.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def delete(self, username): + mailbox = Mailbox.objects.get(name=username) + self.admin_delete(mailbox) + + @snapshot_on_error + def change_password(self, username, password): + mailbox = Mailbox.objects.get(name=username) + self.admin_change_password(mailbox, password) + + @snapshot_on_error + def add_address(self, username, name, domain): + url = self.live_server_url + reverse('admin:mailboxes_address_add') + self.selenium.get(url) + + name_field = self.selenium.find_element_by_id('id_name') + name_field.send_keys(name) + + domain_input = self.selenium.find_element_by_id('id_domain') + domain_select = Select(domain_input) + domain_select.select_by_value(str(domain.pk)) + + mailboxes = self.selenium.find_element_by_id('id_mailboxes_add_all_link') + mailboxes.click() + time.sleep(0.5) + name_field.submit() + + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def delete_address(self, username): + mailbox = Mailbox.objects.get(name=username) + address = mailbox.addresses.get() + self.admin_delete(address) + + @snapshot_on_error + def disable(self, username): + mailbox = Mailbox.objects.get(name=username) + self.admin_disable(mailbox) + + +class RESTMailboxTest(RESTMailboxMixin, BaseLiveServerTestCase): + pass + + +class AdminMailboxTest(AdminMailboxMixin, BaseLiveServerTestCase): + pass diff --git a/orchestra/contrib/mailboxes/validators.py b/orchestra/contrib/mailboxes/validators.py new file mode 100644 index 00000000..5a33b16a --- /dev/null +++ b/orchestra/contrib/mailboxes/validators.py @@ -0,0 +1,70 @@ +import hashlib +import os +import re + +from django.core.validators import ValidationError, EmailValidator +from django.utils.translation import gettext_lazy as _ + +from orchestra.utils import paths +from orchestra.utils.sys import run + +from . import settings + + +def validate_emailname(value): + msg = _("'%s' is not a correct email name." % value) + if '@' in value: + raise ValidationError(msg) + value += '@localhost' + try: + EmailValidator()(value) + except ValidationError: + raise ValidationError(msg) + + +def validate_forward(value): + """ space separated mailboxes or emails """ + from .models import Mailbox + errors = [] + destinations = [] + for destination in value.split(): + if destination in destinations: + errors.append(ValidationError( + _("'%s' is already present.") % destination + )) + destinations.append(destination) + if '@' in destination: + try: + EmailValidator()(destination) + except ValidationError: + errors.append(ValidationError( + _("'%s' is not a valid email address.") % destination + )) + elif not Mailbox.objects.filter(name=destination).exists(): + errors.append(ValidationError( + _("'%s' is not an existent mailbox.") % destination + )) + if errors: + raise ValidationError(errors) + + +def validate_sieve(value): + sieve_name = '%s.sieve' % hashlib.md5(value.encode('utf8')).hexdigest() + test_path = os.path.join(settings.MAILBOXES_SIEVETEST_PATH, sieve_name) + with open(test_path, 'w') as f: + f.write(value) + context = { + 'orchestra_root': paths.get_orchestra_dir() + } + sievetest = settings.MAILBOXES_SIEVETEST_BIN_PATH % context + try: + test = run(' '.join([sievetest, test_path, '/dev/null']), silent=True) + finally: + os.unlink(test_path) + if test.exit_code: + errors = [] + for line in test.stderr.decode('utf8').splitlines(): + error = re.match(r'^.*(line\s+[0-9]+:.*)', line) + if error: + errors += error.groups() + raise ValidationError(' '.join(errors)) diff --git a/orchestra/contrib/mailboxes/widgets.py b/orchestra/contrib/mailboxes/widgets.py new file mode 100644 index 00000000..ad92adc8 --- /dev/null +++ b/orchestra/contrib/mailboxes/widgets.py @@ -0,0 +1,33 @@ +import textwrap + +from django import forms + + +class OpenCustomFilteringOnSelect(forms.Select): + def __init__(self, *args, **kwargs): + collapse = self.get_dynamic_collapse() + attrs = kwargs.get('attrs', {}) + attrs.update({ + 'onClick': collapse, + 'onChange': collapse, + }) + attrs.update(kwargs.get('attrs', {})) + kwargs['attrs'] = attrs + super(OpenCustomFilteringOnSelect, self).__init__(*args, **kwargs) + + def get_dynamic_collapse(self): + return textwrap.dedent("""\ + value = this.options[this.selectedIndex].value; + fieldset = $(this).closest("fieldset"); + fieldset = $(".collapse"); + if ( value == 'CUSTOM' ) { + if (fieldset.hasClass("collapsed")) { + fieldset.removeClass("collapsed").trigger("show.fieldset", [$(this).attr("id")]); + } + } else { + if (! $(this).closest("fieldset").hasClass("collapsed")) { + fieldset.addClass("collapsed").trigger("hide.fieldset", [$(this).attr("id")]); + } + } + """ + ) diff --git a/orchestra/contrib/mailer/README.md b/orchestra/contrib/mailer/README.md new file mode 100644 index 00000000..1b7eae19 --- /dev/null +++ b/orchestra/contrib/mailer/README.md @@ -0,0 +1,5 @@ +This is a simplified clone of [django-mailer](https://github.com/pinax/django-mailer). + +Using `orchestra.contrib.mailer.backends.EmailBackend` as your email backend will have the following effects: + * E-mails sent with Django's `send_mass_mail()` will be queued and sent by an out-of-band perioic task. + * E-mails sent with Django's `send_mail()` will be sent right away by an asynchronous background task. diff --git a/orchestra/contrib/mailer/__init__.py b/orchestra/contrib/mailer/__init__.py new file mode 100644 index 00000000..335e52d3 --- /dev/null +++ b/orchestra/contrib/mailer/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.mailer.apps.MailerConfig' diff --git a/orchestra/contrib/mailer/actions.py b/orchestra/contrib/mailer/actions.py new file mode 100644 index 00000000..1ba1b90a --- /dev/null +++ b/orchestra/contrib/mailer/actions.py @@ -0,0 +1,8 @@ +from django.urls import reverse +from django.shortcuts import redirect + + +def last(modeladmin, request, queryset): + last = queryset.model.objects.latest('id') + url = reverse('admin:mailer_message_change', args=(last.pk,)) + return redirect(url) diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py new file mode 100644 index 00000000..9d508c11 --- /dev/null +++ b/orchestra/contrib/mailer/admin.py @@ -0,0 +1,157 @@ +import base64 +import email + +from django import forms +from django.contrib import admin +from django.urls import reverse +from django.db.models import Count +from django.shortcuts import redirect +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import admin_link, admin_colored, admin_date, wrap_admin_view +from orchestra.contrib.tasks import task + +from .actions import last +from .engine import send_pending +from .models import Message, SMTPLog + + +COLORS = { + Message.QUEUED: 'purple', + Message.SENT: 'green', + Message.DEFERRED: 'darkorange', + Message.FAILED: 'red', + SMTPLog.SUCCESS: 'green', + SMTPLog.FAILURE: 'red', +} + + +class MessageAdmin(ExtendedModelAdmin): + list_display = ( + 'display_subject', 'colored_state', 'priority', 'to_address', 'from_address', + 'created_at_delta', 'display_retries', 'last_try_delta', + ) + list_filter = ('state', 'priority', 'retries') + list_prefetch_related = ('logs',) + search_fields = ('to_address', 'from_address', 'subject',) + fieldsets = ( + (None, { + 'fields': ('state', 'priority', ('retries', 'last_try_delta', 'created_at_delta'), + 'display_full_subject', 'display_from', 'display_to', + 'display_content'), + }), + (_("Edit"), { + 'classes': ('collapse',), + 'fields': ('subject', 'from_address', 'to_address', 'content'), + }), + ) + readonly_fields = ( + 'retries', 'last_try_delta', 'created_at_delta', 'display_full_subject', + 'display_to', 'display_from', 'display_content', + ) + date_hierarchy = 'created_at' + change_view_actions = (last,) + + colored_state = admin_colored('state', colors=COLORS) + created_at_delta = admin_date('created_at') + last_try_delta = admin_date('last_try') + + def display_subject(self, instance): + subject = instance.subject + if len(subject) > 64: + return mark_safe(subject[:64] + '…') + return subject + display_subject.short_description = _("Subject") + display_subject.admin_order_field = 'subject' + + def display_retries(self, instance): + num_logs = instance.logs__count + if num_logs == 1: + pk = instance.logs.all()[0].id + url = reverse('admin:mailer_smtplog_change', args=(pk,)) + else: + url = reverse('admin:mailer_smtplog_changelist') + url += '?&message=%i' % instance.pk + return format_html('{}', url, instance.retries) + display_retries.short_description = _("Retries") + display_retries.admin_order_field = 'retries' + + def display_content(self, instance): + part = email.message_from_string(instance.content) + payload = part.get_payload() + if isinstance(payload, list): + for cpart in payload: + cpayload = cpart.get_payload() + if cpart.get_content_type().startswith('text/'): + part = cpart + payload = cpayload + if cpart.get_content_type() == 'text/html': + payload = '
    %s
    ' % payload + # prioritize HTML + break + if part.get('Content-Transfer-Encoding') == 'base64': + payload = base64.b64decode(payload) + charset = part.get_charsets()[0] + if charset: + payload = payload.decode(charset) + if part.get_content_type() == 'text/plain': + payload = payload.replace('\n', '
    ').replace(' ', ' ') + return mark_safe(payload) + display_content.short_description = _("Content") + + def display_full_subject(self, instance): + return instance.subject + display_full_subject.short_description = _("Subject") + + def display_from(self, instance): + return instance.from_address + display_from.short_description = _("From") + + def display_to(self, instance): + return instance.to_address + display_to.short_description = _("To") + + def get_urls(self): + from django.urls import re_path as url + urls = super().get_urls() + info = self.model._meta.app_label, self.model._meta.model_name + urls.insert(0, + url(r'^send-pending/$', + wrap_admin_view(self, self.send_pending_view), + name='%s_%s_send_pending' % info) + ) + return urls + + def get_queryset(self, request): + qs = super().get_queryset(request) + return qs.annotate(Count('logs')).defer('content') + + def send_pending_view(self, request): + task(send_pending).apply_async() + self.message_user(request, _("Pending messages are being sent on the background.")) + return redirect('..') + + def formfield_for_dbfield(self, db_field, **kwargs): + if db_field.name == 'subject': + kwargs['widget'] = forms.TextInput(attrs={'size':'100'}) + return super().formfield_for_dbfield(db_field, **kwargs) + + +class SMTPLogAdmin(admin.ModelAdmin): + list_display = ( + 'id', 'message_link', 'colored_result', 'date_delta', 'log_message' + ) + list_filter = ('result',) + fields = ('message_link', 'colored_result', 'date_delta', 'log_message') + readonly_fields = fields + + message_link = admin_link('message') + colored_result = admin_colored('result', colors=COLORS, bold=False) + date_delta = admin_date('date') + + +admin.site.register(Message, MessageAdmin) +admin.site.register(SMTPLog, SMTPLogAdmin) diff --git a/orchestra/contrib/mailer/apps.py b/orchestra/contrib/mailer/apps.py new file mode 100644 index 00000000..c680cef8 --- /dev/null +++ b/orchestra/contrib/mailer/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + +from orchestra.core import administration + + +class MailerConfig(AppConfig): + name = 'orchestra.contrib.mailer' + verbose_name = "Mailer" + + def ready(self): + from .models import Message + administration.register(Message, icon='Mail-send.png') diff --git a/orchestra/contrib/mailer/backends.py b/orchestra/contrib/mailer/backends.py new file mode 100644 index 00000000..dd66cafd --- /dev/null +++ b/orchestra/contrib/mailer/backends.py @@ -0,0 +1,53 @@ +from django.conf import settings as djsettings +from django.core.mail import get_connection +from django.core.mail.backends.base import BaseEmailBackend + +from orchestra.core.caches import get_request_cache + +from . import settings +from .models import Message +from .tasks import send_message + + +class EmailBackend(BaseEmailBackend): + """ + A wrapper that manages a queued SMTP system. + """ + def send_messages(self, email_messages): + if not email_messages: + return + # Count messages per request + cache = get_request_cache() + key = 'mailer.sent_messages' + sent_messages = cache.get(key) or 0 + sent_messages += 1 + cache.set(key, sent_messages) + + is_bulk = len(email_messages) > 1 + if sent_messages > settings.MAILER_NON_QUEUED_PER_REQUEST_THRESHOLD: + is_bulk = True + default_priority = Message.NORMAL if is_bulk else Message.CRITICAL + num_sent = 0 + connection = None + for message in email_messages: + priority = message.extra_headers.get('X-Mail-Priority', default_priority) + content = message.message().as_string() + for to_email in message.recipients(): + message = Message( + priority=priority, + to_address=to_email, + from_address=getattr(message, 'from_email', djsettings.DEFAULT_FROM_EMAIL), + subject=message.subject, + content=content, + ) + if priority == Message.CRITICAL: + # send immidiately + if connection is None: + connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend') + send_message.apply_async(message, connection=connection) + else: + message.save() + num_sent += 1 + if connection is not None: + connection.close() + return num_sent diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py new file mode 100644 index 00000000..898f2041 --- /dev/null +++ b/orchestra/contrib/mailer/engine.py @@ -0,0 +1,76 @@ + +import smtplib +from datetime import timedelta +from socket import error as SocketError + +from django.core.mail import get_connection +from django.db.models import Q +from django.utils import timezone + +from orchestra.utils.sys import LockFile, OperationLocked + +from . import settings +from .models import Message + + +def send_message(message, connection=None, bulk=settings.MAILER_BULK_MESSAGES): + message.last_try = timezone.now() + update_fields = ['last_try'] + if message.state != message.QUEUED: + message.retries += 1 + update_fields.append('retries') + message.save(update_fields=update_fields) + if connection is None: + connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend') + if connection.connection is None: + try: + connection.open() + except Exception as err: + message.defer() + message.log(err) + return + error = None + try: + connection.connection.sendmail(message.from_address, [message.to_address], message.content.encode()) + except (SocketError, + smtplib.SMTPSenderRefused, + smtplib.SMTPRecipientsRefused, + smtplib.SMTPAuthenticationError) as err: + message.defer() + error = err + else: + message.sent() + message.log(error) + return connection + + +def send_pending(bulk=settings.MAILER_BULK_MESSAGES): + try: + with LockFile('/dev/shm/mailer.send_pending.lock'): + connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend') + cur, total = 0, 0 + for message in Message.objects.filter(state=Message.QUEUED).order_by('priority', 'last_try', 'created_at'): + if cur >= bulk: + connection.close() + cur = 0 + send_message(message, connection, bulk) + cur += 1 + total += 1 + now = timezone.now() + qs = Q() + for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS): + delta = timedelta(seconds=seconds) + qs = qs | Q(retries=retries, last_try__lte=now-delta) + for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority', 'last_try'): + if cur >= bulk: + connection.close() + cur = 0 + send_message(message, connection, bulk) + cur += 1 + total += 1 + return total + except OperationLocked: + pass + finally: + if 'connection' in vars() and connection.connection is not None: + connection.close() diff --git a/orchestra/contrib/mailer/management/commands/sendpendingmessages.py b/orchestra/contrib/mailer/management/commands/sendpendingmessages.py new file mode 100644 index 00000000..23ba00cf --- /dev/null +++ b/orchestra/contrib/mailer/management/commands/sendpendingmessages.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand + +from orchestra.contrib.tasks.decorators import keep_state + +from ...engine import send_pending + + +class Command(BaseCommand): + help = 'Runs Orchestra method.' + + def handle(self, *args, **options): + keep_state(send_pending)() diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py new file mode 100644 index 00000000..c905768c --- /dev/null +++ b/orchestra/contrib/mailer/models.py @@ -0,0 +1,73 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from . import settings + + +class Message(models.Model): + QUEUED = 'QUEUED' + SENT = 'SENT' + DEFERRED = 'DEFERRED' + FAILED = 'FAILED' + STATES = ( + (QUEUED, _("Queued")), + (SENT, _("Sent")), + (DEFERRED, _("Deferred")), + (FAILED, _("Failed")), + ) + + CRITICAL = 0 + HIGH = 1 + NORMAL = 2 + LOW = 3 + PRIORITIES = ( + (CRITICAL, _("Critical (not queued)")), + (HIGH, _("High")), + (NORMAL, _("Normal")), + (LOW, _("Low")), + ) + + state = models.CharField(_("State"), max_length=16, choices=STATES, default=QUEUED, + db_index=True) + priority = models.PositiveIntegerField(_("Priority"), choices=PRIORITIES, default=NORMAL, + db_index=True) + to_address = models.CharField(max_length=256) + from_address = models.CharField(max_length=256) + subject = models.TextField(_("subject")) + content = models.TextField(_("content")) + created_at = models.DateTimeField(_("created at"), auto_now_add=True) + retries = models.PositiveIntegerField(_("retries"), default=0, db_index=True) + last_try = models.DateTimeField(_("last try"), null=True, db_index=True) + + def __str__(self): + return '%s to %s' % (self.subject, self.to_address) + + def defer(self): + self.state = self.DEFERRED + # Max tries + if self.retries >= len(settings.MAILER_DEFERE_SECONDS): + self.state = self.FAILED + self.save(update_fields=('state',)) + + def sent(self): + self.state = self.SENT + self.save(update_fields=('state',)) + + def log(self, error): + result = SMTPLog.SUCCESS + if error: + result= SMTPLog.FAILURE + self.logs.create(log_message=str(error), result=result) + + +class SMTPLog(models.Model): + SUCCESS = 'SUCCESS' + FAILURE = 'FAILURE' + RESULTS = ( + (SUCCESS, _("Success")), + (FAILURE, _("Failure")), + ) + message = models.ForeignKey(Message, editable=False, related_name='logs', on_delete=models.CASCADE) + result = models.CharField(max_length=16, choices=RESULTS, default=SUCCESS) + date = models.DateTimeField(auto_now_add=True) + log_message = models.TextField() diff --git a/orchestra/contrib/mailer/settings.py b/orchestra/contrib/mailer/settings.py new file mode 100644 index 00000000..e040a523 --- /dev/null +++ b/orchestra/contrib/mailer/settings.py @@ -0,0 +1,24 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting + + +MAILER_DEFERE_SECONDS = Setting('MAILER_DEFERE_SECONDS', + (300, 600, 60*60, 60*60*24), +) + + +MAILER_MESSAGES_CLEANUP_DAYS = Setting('MAILER_MESSAGES_CLEANUP_DAYS', + 7 +) + + +MAILER_NON_QUEUED_PER_REQUEST_THRESHOLD = Setting('MAILER_NON_QUEUED_PER_REQUEST_THRESHOLD', + 2, + help_text=_("Number of emails that will be sent immediately before starting to queue them."), +) + + +MAILER_BULK_MESSAGES = Setting('MAILER_BULK_MESSAGES', + 500, +) diff --git a/orchestra/contrib/mailer/tasks.py b/orchestra/contrib/mailer/tasks.py new file mode 100644 index 00000000..c05d9fed --- /dev/null +++ b/orchestra/contrib/mailer/tasks.py @@ -0,0 +1,23 @@ +from datetime import timedelta + +from django.utils import timezone +from celery.task.schedules import crontab + +from orchestra.contrib.tasks import task, periodic_task + +from . import engine, settings + + +@task +def send_message(message, connection=None): + message.save() + engine.send_message(message, connection=connection) + + +@periodic_task(run_every=crontab(hour=7, minute=30)) +def cleanup_messages(): + from .models import Message + delta = timedelta(days=settings.MAILER_MESSAGES_CLEANUP_DAYS) + now = timezone.now() + epoch = (now-delta) + return Message.objects.filter(state=Message.SENT, created_at__lt=epoch).only('id').delete() diff --git a/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html b/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html new file mode 100644 index 00000000..d39ecb99 --- /dev/null +++ b/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html @@ -0,0 +1,14 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls static admin_list %} + + +{% block object-tools-items %} +
  • + {% url cl.opts|admin_urlname:'send_pending' as send_pending_url %} + + {% blocktrans with cl.opts.verbose_name as name %}Send pending{% endblocktrans %} + +
    • + {{ block.super }} +{% endblock %} + diff --git a/orchestra/contrib/miscellaneous/__init__.py b/orchestra/contrib/miscellaneous/__init__.py new file mode 100644 index 00000000..6294909f --- /dev/null +++ b/orchestra/contrib/miscellaneous/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.miscellaneous.apps.MiscellaneousConfig' diff --git a/orchestra/contrib/miscellaneous/admin.py b/orchestra/contrib/miscellaneous/admin.py new file mode 100644 index 00000000..2fce1981 --- /dev/null +++ b/orchestra/contrib/miscellaneous/admin.py @@ -0,0 +1,150 @@ +from django import forms +from django.contrib import admin +from django.urls import reverse +from django.db import models +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.actions import disable, enable +from orchestra.admin.utils import admin_link +from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter +from orchestra.plugins import PluginModelAdapter +from orchestra.plugins.admin import SelectPluginAdminMixin +from orchestra.utils.python import import_class + +from . import settings +from .models import MiscService, Miscellaneous + + +class MiscServicePlugin(PluginModelAdapter): + model = MiscService + name_field = 'name' + plugin_field = 'service' + + +class MiscServiceAdmin(ExtendedModelAdmin): + list_display = ( + 'display_name', 'display_verbose_name', 'num_instances', 'has_identifier', 'has_amount', 'is_active' + ) + list_editable = ('is_active',) + list_filter = ('has_identifier', 'has_amount', IsActiveListFilter) + fields = ( + 'verbose_name', 'name', 'description', 'has_identifier', 'has_amount', 'is_active' + ) + prepopulated_fields = {'name': ('verbose_name',)} + change_readonly_fields = ('name',) + actions = (disable, enable) + + def display_name(self, misc): + return format_html('{}', misc.description, misc.name) + display_name.short_description = _("name") + display_name.admin_order_field = 'name' + + def display_verbose_name(self, misc): + return format_html('{}', misc.description, misc.verbose_name) + display_verbose_name.short_description = _("verbose name") + display_verbose_name.admin_order_field = 'verbose_name' + + def num_instances(self, misc): + """ return num slivers as a link to slivers changelist view """ + num = misc.instances__count + url = reverse('admin:miscellaneous_miscellaneous_changelist') + url += '?service__name={}'.format(misc.name) + return mark_safe('{1}'.format(url, num)) + num_instances.short_description = _("Instances") + num_instances.admin_order_field = 'instances__count' + + def get_queryset(self, request): + qs = super(MiscServiceAdmin, self).get_queryset(request) + return qs.annotate(models.Count('instances', distinct=True)) + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'description': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 2}) + return super(MiscServiceAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + +class MiscellaneousAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): + list_display = ( + '__str__', 'service_link', 'amount', 'account_link', 'dispaly_active' + ) + list_filter = ('service__name', 'is_active') + list_select_related = ('service', 'account') + readonly_fields = ('account_link', 'service_link') + add_fields = ('service', 'account', 'description', 'is_active') + fields = ('service_link', 'account', 'description', 'is_active') + change_readonly_fields = ('identifier', 'service') + search_fields = ('identifier', 'description', 'account__username') + actions = (disable, enable) + plugin_field = 'service' + plugin = MiscServicePlugin + + service_link = admin_link('service') + + def dispaly_active(self, instance): + return instance.active + dispaly_active.short_description = _("Active") + dispaly_active.boolean = True + dispaly_active.admin_order_field = 'is_active' + + def get_service(self, obj): + if obj is None: + return self.plugin.get(self.plugin_value).related_instance + else: + return obj.service + + def get_fieldsets(self, request, obj=None): + fieldsets = super().get_fieldsets(request, obj) + fields = list(fieldsets[0][1]['fields']) + service = self.get_service(obj) + if obj: + fields.insert(1, 'account_link') + if service.has_amount: + fields.insert(-1, 'amount') + if service.has_identifier: + fields.insert(2, 'identifier') + fieldsets[0][1]['fields'] = fields + return fieldsets + + def get_form(self, request, obj=None, **kwargs): + if obj: + plugin = self.plugin.get(obj.service.name)() + else: + plugin = self.plugin.get(self.plugin_value)() + self.form = plugin.get_form() + self.plugin_instance = plugin + service = self.get_service(obj) + form = super(SelectPluginAdminMixin, self).get_form(request, obj, **kwargs) + def clean_identifier(self, service=service): + identifier = self.cleaned_data['identifier'] + validator_path = settings.MISCELLANEOUS_IDENTIFIER_VALIDATORS.get(service.name, None) + if validator_path: + validator = import_class(validator_path) + validator(identifier) + return identifier + + form.clean_identifier = clean_identifier + return form + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'description': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 4}) + return super(MiscellaneousAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + def save_model(self, request, obj, form, change): + if not change: + plugin = self.plugin + kwargs = { + plugin.name_field: self.plugin_value + } + setattr(obj, self.plugin_field, plugin.model.objects.get(**kwargs)) + obj.save() + + +admin.site.register(MiscService, MiscServiceAdmin) +admin.site.register(Miscellaneous, MiscellaneousAdmin) diff --git a/orchestra/contrib/miscellaneous/apps.py b/orchestra/contrib/miscellaneous/apps.py new file mode 100644 index 00000000..2f5763ae --- /dev/null +++ b/orchestra/contrib/miscellaneous/apps.py @@ -0,0 +1,15 @@ +from django.apps import AppConfig + +from orchestra.core import services, administration +from orchestra.core.translations import ModelTranslation + + +class MiscellaneousConfig(AppConfig): + name = 'orchestra.contrib.miscellaneous' + verbose_name = 'Miscellaneous' + + def ready(self): + from .models import MiscService, Miscellaneous + services.register(Miscellaneous, icon='applications-other.png') + administration.register(MiscService, icon='Misc-Misc-Box-icon.png') + ModelTranslation.register(MiscService, ('verbose_name',)) diff --git a/orchestra/contrib/miscellaneous/models.py b/orchestra/contrib/miscellaneous/models.py new file mode 100644 index 00000000..2a54983c --- /dev/null +++ b/orchestra/contrib/miscellaneous/models.py @@ -0,0 +1,85 @@ +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from orchestra.core.validators import validate_name +from orchestra.models.fields import NullableCharField + + +class MiscService(models.Model): + name = models.CharField(_("name"), max_length=32, unique=True, validators=[validate_name], + help_text=_("Raw name used for internal referenciation, i.e. service match definition")) + verbose_name = models.CharField(_("verbose name"), max_length=256, blank=True, + help_text=_("Human readable name")) + description = models.TextField(_("description"), blank=True, + help_text=_("Optional description")) + has_identifier = models.BooleanField(_("has identifier"), default=True, + help_text=_("Designates if this service has a unique text field that " + "identifies it or not.")) + has_amount = models.BooleanField(_("has amount"), default=False, + help_text=_("Designates whether this service has amount " + "property or not.")) + is_active = models.BooleanField(_("active"), default=True, + help_text=_("Whether new instances of this service can be created " + "or not. Unselect this instead of deleting services.")) + + def __str__(self): + return self.name + + def clean(self): + self.verbose_name = self.verbose_name.strip() + + def get_verbose_name(self): + return self.verbose_name or self.name + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + +class Miscellaneous(models.Model): + service = models.ForeignKey(MiscService, on_delete=models.CASCADE, + verbose_name=_("service"), related_name='instances') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='miscellaneous') + identifier = NullableCharField(_("identifier"), max_length=256, null=True, unique=True, + db_index=True, help_text=_("A unique identifier for this service.")) + description = models.TextField(_("description"), blank=True) + amount = models.PositiveIntegerField(_("amount"), default=1) + is_active = models.BooleanField(_("active"), default=True, + help_text=_("Designates whether this service should be treated as " + "active. Unselect this instead of deleting services.")) + + class Meta: + verbose_name_plural = _("miscellaneous") + + def __str__(self): + return self.identifier or self.description[:32] or str(self.service) + + @cached_property + def active(self): + return self.is_active and self.service.is_active and self.account.is_active + + def get_description(self): + return ' '.join((str(self.amount), self.service.description or self.service.verbose_name)) + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + @cached_property + def service_class(self): + return self.service + + def clean(self): + if self.identifier: + self.identifier = self.identifier.strip().lower() + self.description = self.description.strip() diff --git a/orchestra/contrib/miscellaneous/settings.py b/orchestra/contrib/miscellaneous/settings.py new file mode 100644 index 00000000..bc0ef3e4 --- /dev/null +++ b/orchestra/contrib/miscellaneous/settings.py @@ -0,0 +1,8 @@ +from orchestra.contrib.settings import Setting + + +MISCELLANEOUS_IDENTIFIER_VALIDATORS = Setting('MISCELLANEOUS_IDENTIFIER_VALIDATORS', + { + # : + } +) diff --git a/orchestra/contrib/orchestration/README.md b/orchestra/contrib/orchestration/README.md new file mode 100644 index 00000000..cd87ee86 --- /dev/null +++ b/orchestra/contrib/orchestration/README.md @@ -0,0 +1,80 @@ +# Orchestration + +This module handles the management of the services controlled by Orchestra. This app provides the means for detecting changes on the data model and execute scripts on the servers to reflect those changes. + +Orchestration module has the following pieces: + +* `Operation` encapsulates an operation, storing the related object, the action and the backend +* `OperationsMiddleware` collects and executes all save and delete operations, more on [next section](#operationsmiddleware) +* `manager` it manage the execution of the operations +* `backends` defines the logic that will be executed on the servers in order to control a particular service +* `router` determines in which server an operation should be executed +* `Server` defines a server hosting services +* `methods` script execution methods, e.g. SSH +* `ScriptLog` it logs the script execution + +Routes +====== + +This application provides support for mapping Orchestra service instances to server machines accross the network. + +It supports _routing_ based on Python expression, which means that you can efectively +control services that are distributed accross several machines. For example, different +websites that are distributed accross _n_ web servers on a _shared hosting_ +environment. + +### OperationsMiddleware + +`middlewares.OperationsMiddleware` automatically executes the service backends when a change on the data model occurs. The main steps that performs are: + +1. Collect all `save` and `delete` model signals triggered on each HTTP request +2. Find related backends using the routing backend +3. Generate a single script per server (_unit of work_) +4. Execute the generated scripts on the servers via SSH + + +### Service Management Properties + +We can identify three different characteristics regarding service management: + +* **Authority**: Whether or not Orchestra is the only source of the service configuration. When Orchestra is the authority then service configuration is _completely generated_ from the Orchestra database (or services are configured to read their configuration directly from Orchestra database). Otherwise Orchestra will execute small tasks translating model changes into configuration changes, allowing manual configurations to be preserved. +* **Flow**: _push_, when Orchestra drives the execution or _pull_, when external services connects to Orchestra. +* **Execution**: _synchronous_, when the execution blocks the HTTP request, or _asynchronous_ when it doesn't. Asynchronous execution means concurrency, and concurrency scalability and complexity (i.e. reporting user feedback of success/failed backend executions). + + +### Registry vs Synchronization vs Task +From the above management properties we can extract three main service management strategies: (a) _task based management_, (b) _synchronization based management_ and (c) _registry based management_. Orchestra provides support for all of them, it is left to you to decide which one suits your requirements better. + +Following a brief description and evaluation of the tradeoffs to help on your decision making. + +#### a. Task Based Management (prefered) +This model refers when Orchestra is _not the only source of configuration_. Therefore, Orchestra translates isolated data model changes directly into localized changes on the service configuration, and executing them using a **push** strategy. For example `save()` or `delete()` object-level operations may have sibling configuration management operations. In contrast to _synchronization_, tasks are able to preserve configuration not performed by Orchestra. + +This model is intuitive, efficient and also very consistent when tasks are execute **synchronously** with the request/response cycle. However, **asynchronous** task execution can have _consistency issues_; tasks have state, and this state can be lost when: +- A failure occur while applying some changes, e.g. network error or worker crash while deleting a service +- Scripts are executed out of order, e.g. create and delete a service is applied in inverse order + +In general, _synchornous execution of tasks is preferred_ over asynchornous, unless response delays are not tolerable. + + +#### b. Synchronization Based Management +When Orchestra is the configuration **authority** and also _the responsible of applying the changes_ on the servers (**push** flow). The configuration files are **regenerated** every time by Orchestra, deleting any existing manual configuration. This model is very consistent since it only depends on the current state of the system (_memoryless_). Therefore, it makes sense to execute the synchronization operation in **asynchronous** fashion. + +In contrast to registry based management, synchronization management is _fully centralized_, all the management operations are driven by Orchestra so you don't need to install nor configure anything on your servers. + +#### c. Registry Based Management +When Orchestra acts as a pure **configuration registry (authority)**, doing nothing more than store service's configuration on the database. The configuration is **pulled** from Orchestra by the servers themselves, so it is **asynchronous** by nature. + +This strategy considers two different implementations: + +- The service is configured to read the configuration directly from Orchestra database (or REST API). This approach simplifies configuration management but also can make Orchestra a single point of failure on your architecture. +- An application (_agent_) periodically fetches the service configuration from the Orchestra database and regenerates the service configuration files. This approach is very tolerant to failures, since the services will keep working independenlty from orchestra, and the new configuration will be applied after recovering. A delay may occur until the changes are applied to the services (_eventual consistency_), but it can be mitigated by notifying the application when a relevant change occur. User feedback about the success or failure of appling the configuration needs to be implemented by the agent. + +##### What state does actually mean? +Lets assume you have deleted a mailbox, and Orchestra has created an script that deletes that mailbox on the mail server. However a failure has occurred and the mailbox deletion task has been lost. Since the state has also been lost it is not easy to tell what to do now in order to maintain consistency. + + +### Additional Notes +* The script that manage the service needs to be idempotent, i.e. the outcome of running the script is always the same, no matter how many times it is executed. +* Renaming of attributes may lead to undesirable effects, e.g. changing a database name will create a new database rather than just changing its name. +* The system does not magically perform data migrations between servers when its _route_ has changed diff --git a/orchestra/contrib/orchestration/__init__.py b/orchestra/contrib/orchestration/__init__.py new file mode 100644 index 00000000..c4774481 --- /dev/null +++ b/orchestra/contrib/orchestration/__init__.py @@ -0,0 +1,91 @@ +import collections +import copy + +from orchestra.utils.python import AttrDict + +from .backends import ServiceBackend, ServiceController, replace + + +default_app_config = 'orchestra.contrib.orchestration.apps.OrchestrationConfig' + + +class Operation(): + DELETE = 'delete' + SAVE = 'save' + MONITOR = 'monitor' + EXCEEDED = 'exceeded' + RECOVERY = 'recovery' + + def __str__(self): + return '%s.%s(%s)' % (self.backend, self.action, self.instance) + + def __repr__(self): + return str(self) + + def __hash__(self): + """ set() """ + return hash((self.backend, self.instance, self.action)) + + def __eq__(self, operation): + """ set() """ + return hash(self) == hash(operation) + + def __init__(self, backend, instance, action, routes=None): + self.backend = backend + # instance should maintain any dynamic attribute until backend execution + # deep copy is prefered over copy otherwise objects will share same atributes (queryset cache) + self.instance = copy.deepcopy(instance) + self.action = action + self.routes = routes + + @classmethod + def execute(cls, operations, serialize=False, run_async=None): + from . import manager + scripts, backend_serialize = manager.generate(operations) + return manager.execute(scripts, serialize=(serialize or backend_serialize), run_async=run_async) + + @classmethod + def create_for_action(cls, instances, action): + if not isinstance(instances, collections.Iterable): + instances = [instances] + operations = [] + for instance in instances: + backends = ServiceBackend.get_backends(instance=instance, action=action) + for backend_cls in backends: + operations.append( + cls(backend_cls, instance, action) + ) + return operations + + @classmethod + def execute_action(cls, instances, action): + """ instances can be an object or an iterable for batch processing """ + operations = cls.create_for_action(instances, action) + return cls.execute(operations) + + def preload_context(self): + """ + Heuristic: Running get_context will prevent most of related objects do not exist errors + """ + if self.action == self.DELETE: + if hasattr(self.backend, 'get_context'): + self.backend().get_context(self.instance) + + def store(self, log): + from .models import BackendOperation + return BackendOperation.objects.create( + log=log, + backend=self.backend.get_name(), + instance=self.instance, + action=self.action, + ) + + @classmethod + def load(cls, operation, log=None): + routes = None + if log: + routes = { + (operation.backend, operation.action): AttrDict(host=log.server) + } + return cls(operation.backend_class, operation.instance, operation.action, routes=routes) + diff --git a/orchestra/contrib/orchestration/actions.py b/orchestra/contrib/orchestration/actions.py new file mode 100644 index 00000000..042f19ad --- /dev/null +++ b/orchestra/contrib/orchestration/actions.py @@ -0,0 +1,134 @@ +from collections import defaultdict + +from django.contrib import messages +from django.contrib.admin import helpers +from django.shortcuts import render +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.admin.utils import get_object_from_url, change_url +from orchestra.contrib.orchestration.helpers import message_user +from orchestra.utils.python import OrderedSet + +from . import manager, Operation +from .models import BackendOperation, Route, Server + + +def retry_backend(modeladmin, request, queryset): + related_operations = queryset.values_list('operations__id', flat=True).distinct() + related_operations = BackendOperation.objects.filter(pk__in=related_operations) + related_operations = related_operations.select_related('log__server').prefetch_related('instance') + if request.POST.get('post') == 'generic_confirmation': + operations = [] + for operation in related_operations: + if operation.instance: + op = Operation.load(operation) + operations.append(op) + if not operations: + messages.warning(request, _("No backend operation has been executed.")) + else: + logs = Operation.execute(operations) + message_user(request, logs) + for backendlog in queryset: + modeladmin.log_change(request, backendlog, 'Retried') + return + opts = modeladmin.model._meta + display_objects = [] + deleted_objects = [] + for op in related_operations: + if not op.instance: + deleted_objects.append(op) + else: + context = { + 'backend': op.log.backend, + 'action': op.action, + 'instance': op.instance, + 'instance_url': change_url(op.instance), + 'server': op.log.server, + 'server_url': change_url(op.log.server), + } + display_objects.append(mark_safe( + '%(backend)s.%(action)s(%(instance)s) @ %(server)s' % context + )) + context = { + 'title': _("Are you sure to execute the following backends?"), + 'action_name': _('Retry backend'), + 'action_value': 'retry_backend', + 'display_objects': display_objects, + 'deleted_objects': deleted_objects, + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'obj': get_object_from_url(modeladmin, request), + } + return render(request, 'admin/orchestration/backends/retry.html', context) +retry_backend.short_description = _("Retry") +retry_backend.url_name = 'retry' + + +def orchestrate(modeladmin, request, queryset): + operations = set() + action = Operation.SAVE + operations = OrderedSet() + if queryset.model is Route: + for route in queryset: + routes = [route] + backend = route.backend_class + if action not in backend.actions: + continue + for instance in backend.model_class().objects.all(): + if route.matches(instance): + operations.add(Operation(backend, instance, action, routes=routes)) + elif queryset.model is Server: + models = set() + for server in queryset: + routes = server.routes.all() + for route in routes.filter(is_active=True): + model = route.backend_class.model_class() + models.add(model) + querysets = [model.objects.order_by('id') for model in models] + + route_cache = {} + for model in models: + for instance in model.objects.all(): + manager.collect(instance, action, operations=operations, route_cache=route_cache) + routes = [] + result = [] + for operation in operations: + routes = [route for route in operation.routes if route.host in queryset] + operation.routes = routes + if routes: + result.append(operation) + operations = result + if not operations: + messages.warning(request, _("No related operations.")) + return + + if request.POST.get('post') == 'generic_confirmation': + logs = Operation.execute(operations) + message_user(request, logs) + for obj in queryset: + modeladmin.log_change(request, obj, 'Orchestrated') + return + + opts = modeladmin.model._meta + display_objects = {} + for operation in operations: + try: + display_objects[operation.backend].append(operation) + except KeyError: + display_objects[operation.backend] = [operation] + context = { + 'title': _("Are you sure to execute the following operations?"), + 'action_name': _('Orchestrate'), + 'action_value': 'orchestrate', + 'display_objects': display_objects, + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'obj': get_object_from_url(modeladmin, request), + } + return render(request, 'admin/orchestration/orchestrate.html', context) +orchestrate.help_text = _("Execute all related operations on the server(s)") diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py new file mode 100644 index 00000000..703fba84 --- /dev/null +++ b/orchestra/contrib/orchestration/admin.py @@ -0,0 +1,196 @@ +from django.contrib import admin, messages +from django.utils.html import escape +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangeViewActionsMixin +from orchestra.admin.utils import admin_link, admin_date, admin_colored, display_mono, display_code +from orchestra.plugins.admin import display_plugin_field + +from . import settings, helpers +from .actions import retry_backend, orchestrate +from .backends import ServiceBackend +from .forms import RouteForm +from .models import Server, Route, BackendLog, BackendOperation +from .utils import retrieve_state +from .widgets import RouteBackendSelect + + +STATE_COLORS = { + BackendLog.RECEIVED: 'darkorange', + BackendLog.TIMEOUT: 'red', + BackendLog.STARTED: 'blue', + BackendLog.SUCCESS: 'green', + BackendLog.FAILURE: 'red', + BackendLog.ERROR: 'red', + BackendLog.REVOKED: 'magenta', + BackendLog.NOTHING: 'green', +} + + +class RouteAdmin(ExtendedModelAdmin): + list_display = ( + 'display_backend', 'host', 'match', 'display_model', 'display_actions', 'run_async', + 'is_active' + ) + list_editable = ('host', 'match', 'run_async', 'is_active') + list_filter = ('host', 'is_active', 'run_async', 'backend') + list_prefetch_related = ('host',) + ordering = ('backend',) + add_fields = ('backend', 'host', 'match', 'run_async', 'is_active') + change_form = RouteForm + actions = (orchestrate,) + change_view_actions = actions + + BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends()) + DEFAULT_MATCH = { + backend.get_name(): backend.default_route_match for backend in ServiceBackend.get_backends() + } + + display_backend = display_plugin_field('backend') + + def display_model(self, route): + try: + return route.backend_class.model + except KeyError: + return mark_safe("NOT AVAILABLE") + display_model.short_description = _("model") + + @mark_safe + def display_actions(self, route): + try: + return '
    '.join(route.backend_class.get_actions()) + except KeyError: + return "NOT AVAILABLE" + display_actions.short_description = _("actions") + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Provides dynamic help text on backend form field """ + if db_field.name == 'backend': + kwargs['widget'] = RouteBackendSelect( + 'this.id', self.BACKEND_HELP_TEXT, self.DEFAULT_MATCH) + field = super(RouteAdmin, self).formfield_for_dbfield(db_field, **kwargs) + if db_field.name == 'host': + # Cache host choices + request = kwargs['request'] + choices = getattr(request, '_host_choices_cache', None) + if choices is None: + request._host_choices_cache = choices = list(field.choices) + field.choices = choices + return field + + def get_form(self, request, obj=None, **kwargs): + """ Include dynamic help text for existing objects """ + form = super(RouteAdmin, self).get_form(request, obj, **kwargs) + if obj: + form.base_fields['backend'].help_text = self.BACKEND_HELP_TEXT.get(obj.backend, '') + return form + + def show_orchestration_disabled(self, request): + if settings.ORCHESTRATION_DISABLE_EXECUTION: + msg = _("Orchestration execution is disabled by ORCHESTRATION_DISABLE_EXECUTION setting.") + self.message_user(request, mark_safe(msg), messages.WARNING) + + def changelist_view(self, request, extra_context=None): + self.show_orchestration_disabled(request) + return super(RouteAdmin, self).changelist_view(request, extra_context) + + def changeform_view(self, request, object_id=None, form_url='', extra_context=None): + self.show_orchestration_disabled(request) + return super(RouteAdmin, self).changeform_view( + request, object_id, form_url, extra_context) + + +class BackendOperationInline(admin.TabularInline): + model = BackendOperation + fields = ('action', 'instance_link') + readonly_fields = ('action', 'instance_link') + extra = 0 + can_delete = False + + class Media: + css = { + 'all': ('orchestra/css/hide-inline-id.css',) + } + + def instance_link(self, operation): + link = admin_link('instance')(self, operation) + if link == '---': + return _("Deleted {0}").format(operation.instance_repr or '-'.join( + (escape(operation.content_type), escape(operation.object_id)))) + return link + instance_link.short_description = _("Instance") + + def has_add_permission(self, *args, **kwargs): + return False + + def get_queryset(self, request): + queryset = super(BackendOperationInline, self).get_queryset(request) + return queryset.prefetch_related('instance') + + +class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin): + list_display = ( + 'id', 'backend', 'server_link', 'display_state', 'exit_code', + 'display_created', 'execution_time', + ) + list_display_links = ('id', 'backend') + list_filter = ('state', 'server', 'backend', 'operations__action') + search_fields = ('script',) + date_hierarchy = 'created_at' + inlines = (BackendOperationInline,) + fields = ( + 'backend', 'server_link', 'state', 'display_script', 'mono_stdout', + 'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created', + 'execution_time' + ) + readonly_fields = fields + actions = (retry_backend,) + change_view_actions = actions + + server_link = admin_link('server') + display_created = admin_date('created_at', short_description=_("Created")) + display_state = admin_colored('state', colors=STATE_COLORS) + display_script = display_code('script') + mono_stdout = display_mono('stdout') + mono_stderr = display_mono('stderr') + mono_traceback = display_mono('traceback') + + class Media: + css = { + 'all': ('orchestra/css/pygments/github.css',) + } + + def get_queryset(self, request): + """ Order by structured name and imporve performance """ + qs = super(BackendLogAdmin, self).get_queryset(request) + return qs.select_related('server').defer('script', 'stdout') + + def has_add_permission(self, *args, **kwargs): + return False + + +class ServerAdmin(ExtendedModelAdmin): + list_display = ('name', 'address', 'os', 'display_ping', 'display_uptime') + list_filter = ('os',) + actions = (orchestrate,) + change_view_actions = actions + + def display_ping(self, instance): + return mark_safe(self._remote_state[instance.pk][0]) + display_ping.short_description = _("Ping") + + def display_uptime(self, instance): + return mark_safe(self._remote_state[instance.pk][1]) + display_uptime.short_description = _("Uptime") + + def get_queryset(self, request): + """ Order by structured name and imporve performance """ + qs = super(ServerAdmin, self).get_queryset(request) + if request.method == 'GET' and request.resolver_match.func.__name__ == 'changelist_view': + self._remote_state = retrieve_state(qs) + return qs + +admin.site.register(Server, ServerAdmin) +admin.site.register(BackendLog, BackendLogAdmin) +admin.site.register(Route, RouteAdmin) diff --git a/orchestra/contrib/orchestration/apps.py b/orchestra/contrib/orchestration/apps.py new file mode 100644 index 00000000..6de145dd --- /dev/null +++ b/orchestra/contrib/orchestration/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + +from orchestra.core import administration + + +class OrchestrationConfig(AppConfig): + name = 'orchestra.contrib.orchestration' + verbose_name = "Orchestration" + + def ready(self): + from .models import Server, Route, BackendLog + administration.register(BackendLog, icon='scriptlog.png') + administration.register(Server, parent=BackendLog, icon='vps.png') + administration.register(Route, parent=BackendLog, icon='hal.png') diff --git a/orchestra/contrib/orchestration/backends.py b/orchestra/contrib/orchestration/backends.py new file mode 100644 index 00000000..f1a04fe3 --- /dev/null +++ b/orchestra/contrib/orchestration/backends.py @@ -0,0 +1,249 @@ +import logging +import textwrap +from functools import partial + +from django.apps import apps +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from orchestra import plugins + +from . import methods + +logger = logging.getLogger(__name__) + +def replace(context, pattern, repl): + """ applies replace to all context str values """ + for key, value in context.items(): + if isinstance(value, str): + context[key] = value.replace(pattern, repl) + return context + + +class ServiceMount(plugins.PluginMount): + def __init__(cls, name, bases, attrs): + # Make sure backends specify a model attribute + if not (attrs.get('abstract', False) or name == 'ServiceBackend' or cls.model): + raise AttributeError("'%s' does not have a defined model attribute." % cls) + super(ServiceMount, cls).__init__(name, bases, attrs) + + +class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): + """ + Service management backend base class + + It uses the _unit of work_ design principle, which allows bulk operations to + be conviniently supported. Each backend generates the configuration for all + the changes of all modified objects, reloading the daemon just once. + """ + model = None + related_models = () # ((model, accessor__attribute),) + script_method = methods.SSH + script_executable = '/bin/bash' + function_method = methods.Python + type = 'task' # 'sync' + # Don't wait for the backend to finish before continuing with request/response + ignore_fields = [] + actions = [] + default_route_match = 'True' + # Force the backend manager to block in multiple backend executions executing them synchronously + serialize = False + doc_settings = None + # By default backend will not run if actions do not generate insctructions, + # If your backend uses prepare() or commit() only then you should set force_empty_action_execution = True + force_empty_action_execution = False + + def __str__(self): + return type(self).__name__ + + def __init__(self): + self.head = [] + self.content = [] + self.tail = [] + + def __getattribute__(self, attr): + """ Select head, content or tail section depending on the method name """ + IGNORE_ATTRS = ( + 'append', + 'cmd_section', + 'head', + 'tail', + 'content', + 'script_method', + 'function_method', + 'set_head', + 'set_tail', + 'set_content', + 'actions', + ) + if attr == 'prepare': + self.set_head() + elif attr == 'commit': + self.set_tail() + elif attr not in IGNORE_ATTRS and attr in self.actions: + self.set_content() + return super(ServiceBackend, self).__getattribute__(attr) + + def set_head(self): + self.cmd_section = self.head + + def set_tail(self): + self.cmd_section = self.tail + + def set_content(self): + self.cmd_section = self.content + + @classmethod + def get_actions(cls): + return [ action for action in cls.actions if action in dir(cls) ] + + @classmethod + def get_name(cls): + return cls.__name__ + + @classmethod + def is_main(cls, obj): + opts = obj._meta + return cls.model == '%s.%s' % (opts.app_label, opts.object_name) + + @classmethod + def get_related(cls, obj): + opts = obj._meta + model = '%s.%s' % (opts.app_label, opts.object_name) + logger.debug('Model: {}'.format(model)) + for rel_model, field in cls.related_models: + logger.debug('rel_model: {}'.format(rel_model)) + logger.debug('field: {}'.format(field)) + if rel_model == model: + related = obj + for attribute in field.split('__'): + related = getattr(related, attribute) + if type(related).__name__ == 'RelatedManager': + return related.all() + return [related] + return [] + + @classmethod + def get_backends(cls, instance=None, action=None): + backends = cls.get_plugins() + included = [] + # Filter for instance or action + for backend in backends: + include = True + if instance: + opts = instance._meta + if backend.model != '.'.join((opts.app_label, opts.object_name)): + include = False + if include and action: + if action not in backend.get_actions(): + include = False + if include: + included.append(backend) + return included + + @classmethod + def get_backend(cls, name): + return cls.get(name) + + @classmethod + def model_class(cls): + return apps.get_model(cls.model) + + @property + def scripts(self): + """ group commands based on their method """ + if not self.content: + return [] + scripts = {} + for method, cmd in self.content: + scripts[method] = [] + for method, commands in self.head + self.content + self.tail: + try: + scripts[method] += commands + except KeyError: + pass + return list(scripts.items()) + + def get_banner(self): + now = timezone.localtime(timezone.now()) + time = now.strftime("%h %d, %Y %I:%M:%S %Z") + return "Generated by Orchestra at %s" % time + + def create_log(self, server, **kwargs): + from .models import BackendLog + state = BackendLog.RECEIVED + run = bool(self.scripts) or (self.force_empty_action_execution or bool(self.content)) + if not run: + state = BackendLog.NOTHING + using = kwargs.pop('using', None) + manager = BackendLog.objects + if using: + manager = manager.using(using) + log = manager.create(backend=self.get_name(), state=state, server=server) + return log + + def execute(self, server, run_async=False, log=None): + from .models import BackendLog + if log is None: + log = self.create_log(server) + run = log.state != BackendLog.NOTHING + if run: + scripts = self.scripts + for method, commands in scripts: + method(log, server, commands, run_async) + if log.state != BackendLog.SUCCESS: + break + return log + + def append(self, *cmd): + # aggregate commands acording to its execution method + if isinstance(cmd[0], str): + method = self.script_method + cmd = cmd[0] + else: + method = self.function_method + cmd = partial(*cmd) + if not self.cmd_section or self.cmd_section[-1][0] != method: + self.cmd_section.append((method, [cmd])) + else: + self.cmd_section[-1][1].append(cmd) + + def get_context(self, obj): + return {} + + def prepare(self): + """ + hook for executing something at the beging + define functions or initialize state + """ + self.append(textwrap.dedent("""\ + set -e + set -o pipefail + exit_code=0""") + ) + + def commit(self): + """ + hook for executing something at the end + apply the configuration, usually reloading a service + reloading a service is done in a separated method in order to reload + the service once in bulk operations + """ + self.append('exit $exit_code') + + +class ServiceController(ServiceBackend): + actions = ('save', 'delete') + abstract = True + + @classmethod + def get_verbose_name(cls): + return _("[S] %s") % super(ServiceController, cls).get_verbose_name() + + @classmethod + def get_backends(cls): + """ filter controller classes """ + backends = super(ServiceController, cls).get_backends() + return [ + backend for backend in backends if issubclass(backend, ServiceController) + ] diff --git a/orchestra/contrib/orchestration/forms.py b/orchestra/contrib/orchestration/forms.py new file mode 100644 index 00000000..7ea33538 --- /dev/null +++ b/orchestra/contrib/orchestration/forms.py @@ -0,0 +1,20 @@ +from django import forms + +from orchestra.forms.widgets import SpanWidget, PaddingCheckboxSelectMultiple + + +class RouteForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + super(RouteForm, self).__init__(*args, **kwargs) + if self.instance: + self.fields['backend'].required = False + try: + backend_class = self.instance.backend_class + except KeyError: + self.fields['backend'].widget = SpanWidget( + display='%s NOT AVAILABLE' % self.instance.backend) + else: + self.fields['backend'].widget = SpanWidget() + actions = backend_class.actions + self.fields['async_actions'].widget = PaddingCheckboxSelectMultiple(45) + self.fields['async_actions'].choices = ((action, action) for action in actions) diff --git a/orchestra/contrib/orchestration/helpers.py b/orchestra/contrib/orchestration/helpers.py new file mode 100644 index 00000000..68296ab2 --- /dev/null +++ b/orchestra/contrib/orchestration/helpers.py @@ -0,0 +1,173 @@ +import textwrap + +from django.contrib import messages +from django.core.mail import mail_admins +from django.urls import reverse, NoReverseMatch +from django.utils.html import escape +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra import settings as orchestra_settings +from orchestra.admin.utils import change_url + + +def get_backends_help_text(backends): + help_texts = {} + for backend in backends: + help_text = backend.__doc__ or '' + context = { + 'model': backend.model, + 'related_models': str(backend.related_models), + 'script_executable': backend.script_executable, + 'script_method': '.'.join( + (backend.script_method.__module__, backend.script_method.__name__)), + 'function_method': '.'.join( + (backend.function_method.__module__, backend.function_method.__name__)), + 'actions': str(backend.actions), + } + help_text += textwrap.dedent(""" + - Model: '%(model)s' + - Related models: %(related_models)s + - Script executable: %(script_executable)s + - Script method: %(script_method)s + - Function method: %(function_method)s + - Actions: %(actions)s + """ + ) % context + help_text = help_text.lstrip().splitlines() + help_settings = [''] + if backend.doc_settings: + module, names = backend.doc_settings + for name in names: + value = getattr(module, name) + if isinstance(value, str): + help_settings.append("%s = '%s'" % (name, value)) + else: + help_settings.append("%s = %s" % (name, value)) + help_text += help_settings + help_texts[backend.get_name()] = '
    '.join(help_text) + return help_texts + + +def get_instance_url(operation): + try: + url = change_url(operation.instance) + except NoReverseMatch: + alt_repr = '%s-%s' % (operation.content_type, operation.object_id) + return _("Deleted {0}").format(operation.instance_repr or alt_repr) + return orchestra_settings.ORCHESTRA_SITE_URL + url + + +def send_report(method, args, log): + server = args[0] + backend = method.__self__.__class__.__name__ + subject = '[Orchestra] %s execution %s on %s' % (backend, log.state, server) + separator = "\n%s\n\n" % ('~ '*40,) + operations = '\n'.join( + [' '.join((op.action, get_instance_url(op))) for op in log.operations.all()] + ) + log_url = reverse('admin:orchestration_backendlog_change', args=(log.pk,)) + log_url = orchestra_settings.ORCHESTRA_SITE_URL + log_url + message = separator.join([ + "[EXIT CODE] %s" % log.exit_code, + "[STDERR]\n%s" % log.stderr, + "[STDOUT]\n%s" % log.stdout, + "[SCRIPT]\n%s" % log.script, + "[TRACEBACK]\n%s" % log.traceback, + "[OPERATIONS]\n%s" % operations, + "[BACKEND LOG] %s" % log_url, + ]) + html_message = '\n\n'.join([ + '

    Exit code %s

    ' % log.exit_code, + '

    Stderr

    ' + '
    %s
    ' % escape(log.stderr), + '

    Stdout

    ' + '
    %s
    ' % escape(log.stdout), + '

    Script

    ' + '
    %s
    ' % escape(log.script), + '

    Traceback

    ' + '
    %s
    ' % escape(log.traceback), + '

    Operations

    ' + '
    %s
    ' % escape(operations), + '

    Backend log %s

    ' % (log_url, log_url), + ]) + mail_admins(subject, message, html_message=html_message) + + +def get_backend_url(ids): + if len(ids) == 1: + return reverse('admin:orchestration_backendlog_change', args=ids) + elif len(ids) > 1: + url = reverse('admin:orchestration_backendlog_changelist') + return url + '?id__in=%s' % ','.join(map(str, ids)) + return '' + + +def get_messages(logs): + messages = [] + total, successes, run_async = 0, 0, 0 + ids = [] + async_ids = [] + for log in logs: + total += 1 + try: + # Some EXCEPTION logs are not stored on the database + ids.append(log.pk) + except AttributeError: + pass + if log.is_success: + successes += 1 + elif not log.has_finished: + run_async += 1 + async_ids.append(log.id) + errors = total-successes-run_async + url = get_backend_url(ids) + async_url = get_backend_url(async_ids) + async_msg = '' + if run_async: + async_msg = ngettext( + _('{name} is running on the background'), + _('{run_async} backends are running on the background'), + run_async) + if errors: + if total == 1: + msg = _('{name} has fail to execute') + else: + msg = ngettext( + _('{errors} out of {total} backends has fail to execute'), + _('{errors} out of {total} backends have fail to execute'), + errors) + if async_msg: + msg += ', ' + str(async_msg) + msg = msg.format(errors=errors, run_async=run_async, async_url=async_url, total=total, url=url, + name=log.backend) + messages.append(('error', msg + '.')) + elif successes: + if async_msg: + if total == 1: + msg = _('{name} has been executed') + else: + msg = ngettext( + _('{successes} out of {total} backends has been executed'), + _('{successes} out of {total} backends have been executed'), + successes) + msg += ', ' + str(async_msg) + else: + msg = ngettext( + _('{name} has been executed'), + _('{total} backends have been executed'), + total) + msg = msg.format( + total=total, url=url, async_url=async_url, run_async=run_async, successes=successes, + name=log.backend + ) + messages.append(('success', msg + '.')) + else: + msg = async_msg.format(url=url, async_url=async_url, run_async=run_async, name=log.backend) + messages.append(('success', msg + '.')) + return messages + + +def message_user(request, logs): + for func, msg in get_messages(logs): + getattr(messages, func)(request, mark_safe(msg)) diff --git a/orchestra/contrib/orchestration/management/__init__.py b/orchestra/contrib/orchestration/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/orchestration/management/commands/__init__.py b/orchestra/contrib/orchestration/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/orchestration/management/commands/orchestrate.py b/orchestra/contrib/orchestration/management/commands/orchestrate.py new file mode 100644 index 00000000..211f1b59 --- /dev/null +++ b/orchestra/contrib/orchestration/management/commands/orchestrate.py @@ -0,0 +1,137 @@ +import time +from django.apps import apps +from django.core.management.base import BaseCommand, CommandError +from django.db.models import Q + +from orchestra.contrib.orchestration import manager, Operation +from orchestra.contrib.orchestration.models import Server +from orchestra.contrib.orchestration.backends import ServiceBackend +from orchestra.utils.python import OrderedSet +from orchestra.utils.sys import confirm + + +class Command(BaseCommand): + help = 'Runs orchestration backends.' + + def add_arguments(self, parser): + parser.add_argument('model', nargs='?', + help='Label of a model to execute the orchestration.') + parser.add_argument('query', nargs='*', + help='Query arguments for filter().') + parser.add_argument('--noinput', action='store_false', dest='interactive', default=True, + help='Tells Django to NOT prompt the user for input of any kind.') + parser.add_argument('-a', '--action', action='store', dest='action', + default='save', help='Executes action. Defaults to "save".') + parser.add_argument('-s', '--servers', action='store', dest='servers', + default='', help='Overrides route server resolution with the provided server.') + parser.add_argument('-b', '--backends', action='store', dest='backends', + default='', help='Overrides backend.') + parser.add_argument('-l', '--listbackends', action='store_true', dest='list_backends', default=False, + help='List available baclends.') + parser.add_argument('--dry-run', action='store_true', dest='dry', default=False, + help='Only prints scrtipt.') + + + def collect_operations(self, **options): + model = options.get('model') + backends = options.get('backends') or set() + if backends: + backends = set(backends.split(',')) + servers = options.get('servers') or set() + if servers: + servers = set([Server.objects.get(Q(address=server)|Q(name=server)) for server in servers.split(',')]) + action = options.get('action') + if not model: + models = set() + if servers: + for server in servers: + if backends: + routes = server.routes.filter(backend__in=backends) + else: + routes = server.routes.all() + elif backends: + routes = Route.objects.filter(backend__in=backends) + else: + raise CommandError("Model or --servers or --backends?") + for route in routes.filter(is_active=True): + model = route.backend_class.model_class() + models.add(model) + querysets = [model.objects.order_by('id') for model in models] + else: + kwargs = {} + for comp in options.get('query', []): + comps = iter(comp.split('=')) + for arg in comps: + kwargs[arg] = next(comps).strip().rstrip(',') + model = apps.get_model(*model.split('.')) + queryset = model.objects.filter(**kwargs).order_by('id') + querysets = [queryset] + + operations = OrderedSet() + route_cache = {} + for queryset in querysets: + for instance in queryset: + manager.collect(instance, action, operations=operations, route_cache=route_cache) + if backends: + result = [] + for operation in operations: + if operation.backend in backends: + result.append(operation) + operations = result + if servers: + routes = [] + result = [] + for operation in operations: + routes = [route for route in operation.routes if route.host in servers] + operation.routes = routes + if routes: + result.append(operation) + operations = result + return operations + + def handle(self, *args, **options): + list_backends = options.get('list_backends') + if list_backends: + for backend in ServiceBackend.get_backends(): + self.stdout.write(str(backend).split("'")[1]) + return + interactive = options.get('interactive') + dry = options.get('dry') + operations = self.collect_operations(**options) + scripts, serialize = manager.generate(operations) + servers = set() + # Print scripts + for key, value in scripts.items(): + route, __, __ = key + backend, operations = value + servers.add(str(route.host)) + self.stdout.write('# Execute %s on %s' % (backend.get_name(), route.host)) + for method, commands in backend.scripts: + script = '\n'.join(commands) + self.stdout.write(script.encode('ascii', errors='replace').decode()) + if interactive: + context = { + 'servers': ', '.join(servers), + } + if not confirm("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context): + return + if not dry: + logs = manager.execute(scripts, serialize=serialize, run_async=True) + running = list(logs) + stdout = 0 + stderr = 0 + while running: + for log in running: + cstdout = len(log.stdout) + cstderr = len(log.stderr) + if cstdout > stdout: + self.stdout.write(log.stdout[stdout:]) + stdout = cstdout + if cstderr > stderr: + self.stderr.write(log.stderr[stderr:]) + stderr = cstderr + if log.has_finished: + running.remove(log) + time.sleep(0.05) + for log in logs: + self.stdout.write(' '.join((log.backend, log.state))) diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py new file mode 100644 index 00000000..62572d04 --- /dev/null +++ b/orchestra/contrib/orchestration/manager.py @@ -0,0 +1,209 @@ +import logging +import threading +import traceback +from collections import OrderedDict + +from django.core.mail import mail_admins + +from orchestra.utils import db +from orchestra.utils.python import import_class, OrderedSet + +from . import settings, Operation +from .backends import ServiceBackend +from .helpers import send_report +from .models import BackendLog +from .signals import pre_action, post_action, pre_commit, post_commit, pre_prepare, post_prepare + + +logger = logging.getLogger(__name__) +router = import_class(settings.ORCHESTRATION_ROUTER) + + +def keep_log(execute, log, operations): + def wrapper(*args, **kwargs): + """ send report """ + # Remember that threads have their oun connection poll + # No need to EVER temper with the transaction here + log = kwargs['log'] + try: + log = execute(*args, **kwargs) + except Exception as e: + trace = traceback.format_exc() + log.state = log.EXCEPTION + log.stderr += trace + log.save() + subject = 'EXCEPTION executing backend(s) %s %s' % (args, kwargs) + logger.error(subject) + logger.error(trace) + mail_admins(subject, trace) + # We don't propagate the exception further to avoid transaction rollback + finally: + # Store and log the operation + for operation in operations: + logger.info("Executed %s" % operation) + operation.store(log) + if not log.is_success: + send_report(execute, args, log) + stdout = log.stdout.strip() + stdout and logger.debug('STDOUT %s', stdout.encode('ascii', errors='replace').decode()) + stderr = log.stderr.strip() + stderr and logger.debug('STDERR %s', stderr.encode('ascii', errors='replace').decode()) + return wrapper + + +def generate(operations): + scripts = OrderedDict() + cache = {} + serialize = False + # Generate scripts per route+backend + for operation in operations: + logger.debug("Queued %s" % operation) + if operation.routes is None: + operation.routes = router.objects.get_for_operation(operation, cache=cache) + for route in operation.routes: + # TODO key by action.async + async_action = route.action_is_async(operation.action) + key = (route, operation.backend, async_action) + if key not in scripts: + backend, operations = (operation.backend(), [operation]) + scripts[key] = (backend, operations) + backend.set_head() + pre_prepare.send(sender=backend.__class__, backend=backend) + backend.prepare() + post_prepare.send(sender=backend.__class__, backend=backend) + else: + scripts[key][1].append(operation) + # Get and call backend action method + backend = scripts[key][0] + method = getattr(backend, operation.action) + kwargs = { + 'sender': backend.__class__, + 'backend': backend, + 'instance': operation.instance, + 'action': operation.action, + } + backend.set_content() + pre_action.send(**kwargs) + method(operation.instance) + post_action.send(**kwargs) + if backend.serialize: + serialize = True + for value in scripts.values(): + backend, operations = value + backend.set_tail() + pre_commit.send(sender=backend.__class__, backend=backend) + backend.commit() + post_commit.send(sender=backend.__class__, backend=backend) + return scripts, serialize + + +def execute(scripts, serialize=False, run_async=None): + """ + executes the operations on the servers + + serialize: execute one backend at a time + run_async: do not join threads (overrides route.run_async) + """ + if settings.ORCHESTRATION_DISABLE_EXECUTION: + logger.info('Orchestration execution is dissabled by ORCHESTRATION_DISABLE_EXECUTION.') + return [] + # Execute scripts on each server + executions = [] + threads_to_join = [] + logs = [] + for key, value in scripts.items(): + route, __, async_action = key + backend, operations = value + args = (route.host,) + if run_async is None: + is_async = not serialize and (route.run_async or async_action) + else: + is_async = not serialize and (run_async or async_action) + kwargs = { + 'run_async': is_async, + } + # we clone the connection just in case we are isolated inside a transaction + with db.clone(model=BackendLog) as handle: + log = backend.create_log(*args, using=handle.target) + log._state.db = handle.origin + kwargs['log'] = log + task = keep_log(backend.execute, log, operations) + logger.debug('%s is going to be executed on %s.' % (backend, route.host)) + if serialize: + # Execute one backend at a time, no need for threads + task(*args, **kwargs) + else: + task = db.close_connection(task) + thread = threading.Thread(target=task, args=args, kwargs=kwargs) + thread.start() + if not is_async: + threads_to_join.append(thread) + logs.append(log) + [ thread.join() for thread in threads_to_join ] + return logs + + +def collect(instance, action, **kwargs): + """ collect operations """ + operations = kwargs.get('operations', OrderedSet()) + route_cache = kwargs.get('route_cache', {}) + for backend_cls in ServiceBackend.get_backends(): + # Check if there exists a related instance to be executed for this backend and action + instances = [] + if action in backend_cls.actions: + if backend_cls.is_main(instance): + instances = [(instance, action)] + else: + for candidate in backend_cls.get_related(instance): + if candidate.__class__.__name__ == 'ManyRelatedManager': + if 'pk_set' in kwargs: + # m2m_changed signal + candidates = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + else: + candidates = candidate.all() + else: + candidates = [candidate] + for candidate in candidates: + # Check if a delete for candidate is in operations + delete_mock = Operation(backend_cls, candidate, Operation.DELETE) + if delete_mock not in operations: + # related objects with backend.model trigger save() + instances.append((candidate, Operation.SAVE)) + for selected, iaction in instances: + # Maintain consistent state of operations based on save/delete behaviour + # Prevent creating a deleted selected by deleting existing saves + if iaction == Operation.DELETE: + save_mock = Operation(backend_cls, selected, Operation.SAVE) + try: + operations.remove(save_mock) + except KeyError: + pass + else: + update_fields = kwargs.get('update_fields', None) + if update_fields is not None: + # TODO remove this, django does not execute post_save if update_fields=[]... + # Maybe open a ticket at Djangoproject ? + # INITIAL INTENTION: "update_fields=[]" is a convention for explicitly executing backend + # i.e. account.disable() + if update_fields != []: + execute = False + for field in update_fields: + if field not in backend_cls.ignore_fields: + execute = True + break + if not execute: + continue + operation = Operation(backend_cls, selected, iaction) + # Only schedule operations if the router has execution routes + routes = router.objects.get_for_operation(operation, cache=route_cache) + if routes: + logger.debug("Operation %s collected for execution" % operation) + operation.routes = routes + if iaction != Operation.DELETE: + # usually we expect to be using last object state, + # except when we are deleting it + operations.discard(operation) + elif iaction == Operation.DELETE: + operation.preload_context() + operations.add(operation) + return operations diff --git a/orchestra/contrib/orchestration/managers.py b/orchestra/contrib/orchestration/managers.py new file mode 100644 index 00000000..f91ae291 --- /dev/null +++ b/orchestra/contrib/orchestration/managers.py @@ -0,0 +1,81 @@ +import sys +from threading import local + +from django.contrib.admin.models import LogEntry +from django.db.models.signals import pre_delete, post_save, m2m_changed +from django.dispatch import receiver +from django.utils.decorators import ContextDecorator + +from orchestra.utils.python import OrderedSet + +from . import manager, Operation, helpers +from .middlewares import OperationsMiddleware +from .models import BackendLog, BackendOperation + + +@receiver(post_save, dispatch_uid='orchestration.post_save_manager_collector') +def post_save_collector(sender, *args, **kwargs): + if sender not in (BackendLog, BackendOperation, LogEntry): + instance = kwargs.get('instance') + orchestrate.collect(Operation.SAVE, **kwargs) + + +@receiver(pre_delete, dispatch_uid='orchestration.pre_delete_manager_collector') +def pre_delete_collector(sender, *args, **kwargs): + if sender not in (BackendLog, BackendOperation, LogEntry): + orchestrate.collect(Operation.DELETE, **kwargs) + + +@receiver(m2m_changed, dispatch_uid='orchestration.m2m_manager_collector') +def m2m_collector(sender, *args, **kwargs): + # m2m relations without intermediary models are shit. Model.post_save is not sent and + # by the time related.post_save is sent rel objects are not accessible via RelatedManager.all() + if kwargs.pop('action') == 'post_add' and kwargs['pk_set']: + orchestrate.collect(Operation.SAVE, **kwargs) + + +class orchestrate(ContextDecorator): + """ + Context manager for triggering backend operations out of request-response cycle, e.g. shell + + with orchestrate(): + user = SystemUser.objects.get(username='rata') + user.shell = '/dev/null' + user.save(update_fields=('shell',)) + """ + thread_locals = local() + thread_locals.pending_operations = None + thread_locals.route_cache = None + + @classmethod + def collect(cls, action, **kwargs): + """ Collects all pending operations derived from model signals """ + if cls.thread_locals.pending_operations is None: + # No active orchestrate context manager + return + kwargs['operations'] = cls.thread_locals.pending_operations + kwargs['route_cache'] = cls.thread_locals.route_cache + instance = kwargs.pop('instance') + manager.collect(instance, action, **kwargs) + + def __enter__(self): + cls = type(self) + self.old_pending_operations = cls.thread_locals.pending_operations + cls.thread_locals.pending_operations = OrderedSet() + self.old_route_cache = cls.thread_locals.route_cache + cls.thread_locals.route_cache = {} + + def __exit__(self, exc_type, exc_value, traceback): + cls = type(self) + if not exc_type: + operations = cls.thread_locals.pending_operations + if operations: + scripts, serialize = manager.generate(operations) + logs = manager.execute(scripts, serialize=serialize) + for t, msg in helpers.get_messages(logs): + if t == 'error': + sys.stderr.write('%s: %s\n' % (t, msg)) + else: + sys.stdout.write('%s: %s\n' % (t, msg)) + cls.thread_locals.pending_operations = self.old_pending_operations + cls.thread_locals.route_cache = self.old_route_cache diff --git a/orchestra/contrib/orchestration/methods.py b/orchestra/contrib/orchestration/methods.py new file mode 100644 index 00000000..cd3d7a22 --- /dev/null +++ b/orchestra/contrib/orchestration/methods.py @@ -0,0 +1,186 @@ +import inspect +import logging +import socket +import sys +import select +import textwrap + +from celery.datastructures import ExceptionInfo + +from orchestra.settings import ORCHESTRA_SSH_DEFAULT_USER +from orchestra.utils.sys import sshrun +from orchestra.utils.python import CaptureStdout, import_class + +from . import settings + + +logger = logging.getLogger(__name__) + + +def Paramiko(backend, log, server, cmds, run_async=False, paramiko_connections={}): + """ + Executes cmds to remote server using Pramaiko + """ + import paramiko + script = '\n'.join(cmds) + script = script.replace('\r', '') + log.state = log.STARTED + log.script = script + log.save(update_fields=('script', 'state', 'updated_at')) + if not cmds: + return + channel = None + ssh = None + try: + addr = server.get_address() + # ssh connection + ssh = paramiko_connections.get(addr) + if not ssh: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + key = settings.ORCHESTRATION_SSH_KEY_PATH + try: + ssh.connect(addr, username=ORCHESTRA_SSH_DEFAULT_USER, key_filename=key) + except socket.error as e: + logger.error('%s timed out on %s' % (backend, addr)) + log.state = log.TIMEOUT + log.stderr = str(e) + log.save(update_fields=('state', 'stderr', 'updated_at')) + return + paramiko_connections[addr] = ssh + transport = ssh.get_transport() + channel = transport.open_session() + channel.exec_command(backend.script_executable) + channel.sendall(script) + channel.shutdown_write() + # Log results + logger.debug('%s running on %s' % (backend, server)) + if run_async: + second = False + while True: + # Non-blocking is the secret ingridient in the async sauce + select.select([channel], [], []) + if channel.recv_ready(): + part = channel.recv(1024).decode('utf-8') + while part: + log.stdout += part + part = channel.recv(1024).decode('utf-8') + if channel.recv_stderr_ready(): + part = channel.recv_stderr(1024).decode('utf-8') + while part: + log.stderr += part + part = channel.recv_stderr(1024).decode('utf-8') + log.save(update_fields=('stdout', 'stderr', 'updated_at')) + if channel.exit_status_ready(): + if second: + break + second = True + else: + log.stdout += channel.makefile('rb', -1).read().decode('utf-8') + log.stderr += channel.makefile_stderr('rb', -1).read().decode('utf-8') + + log.exit_code = channel.recv_exit_status() + log.state = log.SUCCESS if log.exit_code == 0 else log.FAILURE + logger.debug('%s execution state on %s is %s' % (backend, server, log.state)) + log.save() + except: + log.state = log.ERROR + log.traceback = ExceptionInfo(sys.exc_info()).traceback + logger.error('Exception while executing %s on %s' % (backend, server)) + logger.debug(log.traceback) + log.save() + finally: + if log.state == log.STARTED: + log.state = log.ABORTED + log.save(update_fields=('state', 'updated_at')) + if channel is not None: + channel.close() + + +def OpenSSH(backend, log, server, cmds, run_async=False): + """ + Executes cmds to remote server using SSH with connection resuse for maximum performance + """ + script = '\n'.join(cmds) + script = script.replace('\r', '') + log.state = log.STARTED + log.script = '\n'.join((log.script, script)) + log.save(update_fields=('script', 'state', 'updated_at')) + if not cmds: + return + try: + ssh = sshrun(server.get_address(), script, executable=backend.script_executable, + persist=True, run_async=run_async, silent=True) + logger.debug('%s running on %s' % (backend, server)) + if run_async: + for state in ssh: + log.stdout += state.stdout.decode('utf8') + log.stderr += state.stderr.decode('utf8') + log.save(update_fields=('stdout', 'stderr', 'updated_at')) + exit_code = state.exit_code + else: + log.stdout += ssh.stdout.decode('utf8') + log.stderr += ssh.stderr.decode('utf8') + exit_code = ssh.exit_code + if not log.exit_code: + log.exit_code = exit_code + if exit_code == 255 and log.stderr.startswith('ssh: connect to host'): + log.state = log.TIMEOUT + else: + log.state = log.SUCCESS if exit_code == 0 else log.FAILURE + logger.debug('%s execution state on %s is %s' % (backend, server, log.state)) + log.save() + except: + log.state = log.ERROR + log.traceback = ExceptionInfo(sys.exc_info()).traceback + logger.error('Exception while executing %s on %s' % (backend, server)) + logger.debug(log.traceback) + log.save() + finally: + if log.state == log.STARTED: + log.state = log.ABORTED + log.save(update_fields=('state', 'updated_at')) + + +def SSH(*args, **kwargs): + """ facade function enabling to chose between multiple SSH backends""" + method = import_class(settings.ORCHESTRATION_SSH_METHOD_BACKEND) + return method(*args, **kwargs) + + +def Python(backend, log, server, cmds, run_async=False): + script = '' + functions = set() + for cmd in cmds: + if cmd.func not in functions: + functions.add(cmd.func) + script += textwrap.dedent(''.join(inspect.getsourcelines(cmd.func)[0])) + script += '\n' + for cmd in cmds: + script += '# %s %s\n' % (cmd.func.__name__, cmd.args) + log.state = log.STARTED + log.script = '\n'.join((log.script, script)) + log.save(update_fields=('script', 'state', 'updated_at')) + stdout = '' + try: + for cmd in cmds: + with CaptureStdout() as stdout: + result = cmd(server) + for line in stdout: + log.stdout += line + '\n' + if result: + log.stdout += '# Result: %s\n' % result + if run_async: + log.save(update_fields=('stdout', 'updated_at')) + except: + log.exit_code = 1 + log.state = log.FAILURE + log.stdout += '\n'.join(stdout) + log.traceback += ExceptionInfo(sys.exc_info()).traceback + logger.error('Exception while executing %s on %s' % (backend, server)) + else: + if not log.exit_code: + log.exit_code = 0 + log.state = log.SUCCESS + logger.debug('%s execution state on %s is %s' % (backend, server, log.state)) + log.save() diff --git a/orchestra/contrib/orchestration/middlewares.py b/orchestra/contrib/orchestration/middlewares.py new file mode 100644 index 00000000..61333c56 --- /dev/null +++ b/orchestra/contrib/orchestration/middlewares.py @@ -0,0 +1,116 @@ +from threading import local + +from django.contrib.admin.models import LogEntry +from django.db import transaction +from django.db.models.signals import m2m_changed, post_save, pre_delete +from django.dispatch import receiver +from django.http.response import HttpResponseServerError +from django.urls import resolve +from django.utils.deprecation import MiddlewareMixin +from orchestra.utils.python import OrderedSet + +from . import Operation, manager +from .helpers import message_user +from .models import BackendLog, BackendOperation + + +@receiver(post_save, dispatch_uid='orchestration.post_save_collector') +def post_save_collector(sender, *args, **kwargs): + if sender not in (BackendLog, BackendOperation, LogEntry): + instance = kwargs.get('instance') + OperationsMiddleware.collect(Operation.SAVE, **kwargs) + + +@receiver(pre_delete, dispatch_uid='orchestration.pre_delete_collector') +def pre_delete_collector(sender, *args, **kwargs): + if sender not in (BackendLog, BackendOperation, LogEntry): + OperationsMiddleware.collect(Operation.DELETE, **kwargs) + + +@receiver(m2m_changed, dispatch_uid='orchestration.m2m_collector') +def m2m_collector(sender, *args, **kwargs): + # m2m relations without intermediary models are shit. Model.post_save is not sent and + # by the time related.post_save is sent rel objects are not accessible via RelatedManager.all() + if kwargs.pop('action') == 'post_add' and kwargs['pk_set']: + OperationsMiddleware.collect(Operation.SAVE, **kwargs) + + +class OperationsMiddleware(MiddlewareMixin): + """ + Stores all the operations derived from save and delete signals and executes them + at the end of the request/response cycle + + It also works as a transaction middleware, making requets to run within an atomic block. + """ + # Thread local is used because request object is not available on model signals + thread_locals = local() + + @classmethod + def get_pending_operations(cls): + # Check if an error poped up before OperationsMiddleware.process_request() + if hasattr(cls.thread_locals, 'request'): + request = cls.thread_locals.request + if not hasattr(request, 'pending_operations'): + request.pending_operations = OrderedSet() + return request.pending_operations + return set() + + @classmethod + def get_route_cache(cls): + """ chache the routes to save sql queries """ + if hasattr(cls.thread_locals, 'request'): + request = cls.thread_locals.request + if not hasattr(request, 'route_cache'): + request.route_cache = {} + return request.route_cache + return {} + + @classmethod + def collect(cls, action, **kwargs): + """ Collects all pending operations derived from model signals """ + request = getattr(cls.thread_locals, 'request', None) + if request is None: + return + kwargs['operations'] = cls.get_pending_operations() + kwargs['route_cache'] = cls.get_route_cache() + instance = kwargs.pop('instance') + manager.collect(instance, action, **kwargs) + + def enter_transaction_management(self): + type(self).thread_locals.transaction = transaction.atomic() + type(self).thread_locals.transaction.__enter__() + + def leave_transaction_management(self, exception=None): + locals = type(self).thread_locals + if hasattr(locals, 'transaction'): + # Don't fucking know why sometimes thread_locals does not contain a transaction + locals.transaction.__exit__(exception, None, None) + + def process_request(self, request): + """ Store request on a thread local variable """ + type(self).thread_locals.request = request + self.enter_transaction_management() + + def process_exception(self, request, exception): + """Rolls back the database and leaves transaction management""" + self.leave_transaction_management(exception) + + def process_response(self, request, response): + """ Processes pending backend operations """ + if response.status_code != 500: + operations = self.get_pending_operations() + if operations: + try: + scripts, serialize = manager.generate(operations) + except Exception as exception: + self.leave_transaction_management(exception) + raise + # We commit transaction just before executing operations + # because here is when IntegrityError show up + self.leave_transaction_management() + logs = manager.execute(scripts, serialize=serialize) + if logs and resolve(request.path).app_name == 'admin': + message_user(request, logs) + return response + self.leave_transaction_management() + return response diff --git a/orchestra/contrib/orchestration/models.py b/orchestra/contrib/orchestration/models.py new file mode 100644 index 00000000..4f8606d8 --- /dev/null +++ b/orchestra/contrib/orchestration/models.py @@ -0,0 +1,266 @@ +import logging +import socket + +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.encoding import force_str +from django.utils.functional import cached_property +from django.utils.module_loading import autodiscover_modules +from django.utils.translation import gettext_lazy as _ + +from orchestra.core.validators import validate_ip_address, validate_hostname, OrValidator +from orchestra.models.fields import NullableCharField, MultiSelectField + +from . import settings +from .backends import ServiceBackend + + +logger = logging.getLogger(__name__) + + +class Server(models.Model): + """ Machine runing daemons (services) """ + name = models.CharField(_("name"), max_length=256, unique=True, + help_text=_("Verbose name or hostname of this server.")) + address = NullableCharField(_("address"), max_length=256, blank=True, + validators=[OrValidator(validate_ip_address, validate_hostname)], + null=True, unique=True, help_text=_( + "Optional IP address or domain name. If blank, name field will be used for address resolution.
    " + "If the IP address never changes you can set this field and save DNS requests.")) + description = models.TextField(_("description"), blank=True) + os = models.CharField(_("operative system"), max_length=32, + choices=settings.ORCHESTRATION_OS_CHOICES, + default=settings.ORCHESTRATION_DEFAULT_OS) + + def __str__(self): + return self.name or str(self.address) + + def get_address(self): + if self.address: + return self.address + return self.name + + def get_ip(self): + address = self.get_address() + try: + return validate_ip_address(address) + except ValidationError: + return socket.gethostbyname(self.name) + + def clean(self): + self.name = self.name.strip() + if self.address: + self.address = self.address.strip() + elif self.name: + validate = OrValidator(validate_ip_address, validate_hostname) + validate_hostname(self.name) + try: + validate(self.name) + except ValidationError as err: + raise ValidationError({ + 'name': _("Name should be a valid hostname or IP address when address is not provided.") + }) + + +class BackendLog(models.Model): + RECEIVED = 'RECEIVED' + TIMEOUT = 'TIMEOUT' + STARTED = 'STARTED' + SUCCESS = 'SUCCESS' + FAILURE = 'FAILURE' + ERROR = 'ERROR' + REVOKED = 'REVOKED' + ABORTED = 'ABORTED' + NOTHING = 'NOTHING' + # Special state for mocked backendlogs + EXCEPTION = 'EXCEPTION' + + STATES = ( + (RECEIVED, RECEIVED), + (TIMEOUT, TIMEOUT), + (STARTED, STARTED), + (SUCCESS, SUCCESS), + (FAILURE, FAILURE), + (ERROR, ERROR), + (ABORTED, ABORTED), + (REVOKED, REVOKED), + (NOTHING, NOTHING), + ) + + backend = models.CharField(_("backend"), max_length=256) + state = models.CharField(_("state"), max_length=16, choices=STATES, default=RECEIVED) + server = models.ForeignKey(Server, verbose_name=_("server"), related_name='execution_logs', on_delete=models.CASCADE) + script = models.TextField(_("script")) + stdout = models.TextField(_("stdout")) + stderr = models.TextField(_("stderr")) + traceback = models.TextField(_("traceback")) + exit_code = models.IntegerField(_("exit code"), null=True) + task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True, + help_text="Celery task ID when used as execution backend") + created_at = models.DateTimeField(_("created"), auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(_("updated"), auto_now=True) + + class Meta: + get_latest_by = 'id' + + def __str__(self): + return "%s@%s" % (self.backend, self.server) + + @property + def execution_time(self): + return (self.updated_at-self.created_at).total_seconds() + + @property + def has_finished(self): + return self.state not in (self.STARTED, self.RECEIVED) + + @property + def is_success(self): + return self.state in (self.SUCCESS, self.NOTHING) + + def backend_class(self): + return ServiceBackend.get_backend(self.backend) + + +class BackendOperationQuerySet(models.QuerySet): + def create(self, **kwargs): + instance = kwargs.get('instance') + if instance and 'instance_repr' not in kwargs: + kwargs['instance_repr'] = force_str(instance)[:256] + return super(BackendOperationQuerySet, self).create(**kwargs) + + +class BackendOperation(models.Model): + """ + Encapsulates an operation, storing its related object, the action and the backend. + """ + log = models.ForeignKey('orchestration.BackendLog', related_name='operations', on_delete=models.CASCADE) + backend = models.CharField(_("backend"), max_length=256) + action = models.CharField(_("action"), max_length=64) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + object_id = models.PositiveIntegerField(null=True) + instance_repr = models.CharField(_("instance representation"), max_length=256) + + instance = GenericForeignKey('content_type', 'object_id') + objects = BackendOperationQuerySet.as_manager() + + class Meta: + verbose_name = _("Operation") + verbose_name_plural = _("Operations") + index_together = ( + ('content_type', 'object_id'), + ) + + def __str__(self): + return '%s.%s(%s)' % (self.backend, self.action, self.instance or self.instance_repr) + + @cached_property + def backend_class(self): + return ServiceBackend.get_backend(self.backend) + + +autodiscover_modules('backends') + + +class RouteQuerySet(models.QuerySet): + def get_for_operation(self, operation, **kwargs): + cache = kwargs.get('cache', {}) + if not cache: + for route in self.filter(is_active=True).select_related('host'): + try: + backend_class = route.backend_class + except KeyError: + logger.warning("Backed '%s' not installed." % route.backend) + else: + for action in backend_class.get_actions(): + key = (route.backend, action) + try: + cache[key].append(route) + except KeyError: + cache[key] = [route] + routes = [] + backend_cls = operation.backend + key = (backend_cls.get_name(), operation.action) + try: + target_routes = cache[key] + except KeyError: + pass + else: + for route in target_routes: + if route.matches(operation.instance): + routes.append(route) + return routes + + +class Route(models.Model): + """ + Defines the routing that determine in which server a backend is executed + """ + backend = models.CharField(_("backend"), max_length=256, + choices=ServiceBackend.get_choices()) + host = models.ForeignKey(Server, verbose_name=_("host"), related_name='routes', on_delete=models.CASCADE) + match = models.CharField(_("match"), max_length=256, blank=True, default='True', + help_text=_("Python expression used for selecting the targe host, " + "instance referes to the current object.")) + run_async = models.BooleanField(default=False, + help_text=_("Whether or not block the request/response cycle waitting this backend to " + "finish its execution. Usually you want slave servers to run asynchronously.")) + async_actions = MultiSelectField(max_length=256, blank=True, choices=[], + help_text=_("Specify individual actions to be executed asynchronoulsy.")) +# method = models.CharField(_("method"), max_lenght=32, choices=method_choices, +# default=MethodBackend.get_default()) + is_active = models.BooleanField(_("active"), default=True) + + objects = RouteQuerySet.as_manager() + + class Meta: + unique_together = ('backend', 'host') + + def __str__(self): + return "%s@%s" % (self.backend, self.host) + + @cached_property + def backend_class(self): + return ServiceBackend.get_backend(self.backend) + + def clean(self): + if not self.match: + self.match = 'True' + if self.backend: + try: + backend_class = self.backend_class + except KeyError: + raise ValidationError({ + 'backend': _("Backend '%s' is not installed.") % self.backend + }) + backend_model = backend_class.model_class() + try: + obj = backend_model.objects.all()[0] + except IndexError: + return + try: + bool(self.matches(obj)) + except Exception as exception: + name = type(exception).__name__ + raise ValidationError(': '.join((name, str(exception)))) + + def action_is_async(self, action): + return action in self.async_actions + + def matches(self, instance): + safe_locals = { + 'instance': instance, + 'obj': instance, + instance._meta.model_name: instance, + } + return eval(self.match, safe_locals) + + def enable(self): + self.is_active = True + self.save() + + def disable(self): + self.is_active = False + self.save() diff --git a/orchestra/contrib/orchestration/settings.py b/orchestra/contrib/orchestration/settings.py new file mode 100644 index 00000000..fee41c42 --- /dev/null +++ b/orchestra/contrib/orchestration/settings.py @@ -0,0 +1,51 @@ +from os import path + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting + + +ORCHESTRATION_OS_CHOICES = Setting('ORCHESTRATION_OS_CHOICES', + ( + ('LINUX', "Linux"), + ), + validators=[Setting.validate_choices] +) + + +ORCHESTRATION_DEFAULT_OS = Setting('ORCHESTRATION_DEFAULT_OS', + 'LINUX', + choices=ORCHESTRATION_OS_CHOICES +) + + +ORCHESTRATION_SSH_KEY_PATH = Setting('ORCHESTRATION_SSH_KEY_PATH', + path.join(path.expanduser('~'), '.ssh/id_rsa') +) + + +ORCHESTRATION_ROUTER = Setting('ORCHESTRATION_ROUTER', + 'orchestra.contrib.orchestration.models.Route', + validators=[Setting.validate_import_class] +) + + + +ORCHESTRATION_DISABLE_EXECUTION = Setting('ORCHESTRATION_DISABLE_EXECUTION', + False +) + + +ORCHESTRATION_BACKEND_CLEANUP_DAYS = Setting('ORCHESTRATION_BACKEND_CLEANUP_DAYS', + 20 +) + + +ORCHESTRATION_SSH_METHOD_BACKEND = Setting('ORCHESTRATION_SSH_METHOD_BACKEND', + 'orchestra.contrib.orchestration.methods.OpenSSH', + help_text=_("Two methods are provided:
    " + "1) orchestra.contrib.orchestration.methods.OpenSSH with ControlPersist.
    " + "2) orchestra.contrib.orchestration.methods.Paramiko with connection pool.
    " + "Both perform similarly, but OpenSSH has the advantage that the connections are shared between workers. " + "Paramiko, in contrast, has a per worker connection pool.") +) diff --git a/orchestra/contrib/orchestration/signals.py b/orchestra/contrib/orchestration/signals.py new file mode 100644 index 00000000..a49f493b --- /dev/null +++ b/orchestra/contrib/orchestration/signals.py @@ -0,0 +1,14 @@ +import django.dispatch + +pre_action = django.dispatch.Signal() + +post_action = django.dispatch.Signal() + +pre_prepare = django.dispatch.Signal() + +post_prepare = django.dispatch.Signal() + +pre_commit = django.dispatch.Signal() + +post_commit = django.dispatch.Signal() + diff --git a/orchestra/contrib/orchestration/tasks.py b/orchestra/contrib/orchestration/tasks.py new file mode 100644 index 00000000..35146c2c --- /dev/null +++ b/orchestra/contrib/orchestration/tasks.py @@ -0,0 +1,16 @@ +from datetime import timedelta + +from celery.task.schedules import crontab +from django.utils import timezone + +from orchestra.contrib.tasks import periodic_task + +from . import settings +from .models import BackendLog + + +@periodic_task(run_every=crontab(hour=7, minute=0)) +def backend_logs_cleanup(): + days = settings.ORCHESTRATION_BACKEND_CLEANUP_DAYS + epoch = timezone.now()-timedelta(days=days) + return BackendLog.objects.filter(created_at__lt=epoch).only('id').delete() diff --git a/orchestra/contrib/orchestration/templates/admin/orchestration/backends/retry.html b/orchestra/contrib/orchestration/templates/admin/orchestration/backends/retry.html new file mode 100644 index 00000000..65f2147b --- /dev/null +++ b/orchestra/contrib/orchestration/templates/admin/orchestration/backends/retry.html @@ -0,0 +1,9 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} + + +{% block form %} + {% if deleted_objects %} +

    The following operations refere to deleted objects and will not be executed

    +
      {{ deleted_objects | unordered_list }}
    + {% endif %} +{% endblock %} diff --git a/orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html b/orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html new file mode 100644 index 00000000..df43832b --- /dev/null +++ b/orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html @@ -0,0 +1,16 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} +{% load utils %} + +{% block display_objects %} +
      + {% for backend, operations in display_objects.items %} +
    • Backend: {{ backend }} +
        + {% for operation in operations %} +
      • {{ operation.instance|admin_link }} @ {% for route in operation.routes %}{{ route.host|admin_link }}{% if not forloop.last %},{% endif %} {% endfor %}
        • + {% endfor %} +
      +
      • + {% endfor %} +
    +{% endblock %} diff --git a/orchestra/contrib/orchestration/tests/__init__.py b/orchestra/contrib/orchestration/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/orchestration/tests/test_route.py b/orchestra/contrib/orchestration/tests/test_route.py new file mode 100644 index 00000000..390661ee --- /dev/null +++ b/orchestra/contrib/orchestration/tests/test_route.py @@ -0,0 +1,41 @@ +from orchestra.utils.tests import BaseTestCase + +from .. import backends, Operation +from ..models import Route, Server + + +class RouterTests(BaseTestCase): + def setUp(self): + self.host = Server.objects.create(name='web.example.com') + self.host1 = Server.objects.create(name='web1.example.com') + self.host2 = Server.objects.create(name='web2.example.com') + + def test_list_backends(self): + # TODO count actual, register and compare + choices = list(Route._meta.get_field('backend').choices) + self.assertLess(1, len(choices)) + + def test_get_instances(self): + + class TestBackend(backends.ServiceController): + verbose_name = 'Route' + models = ['routes.Route'] + + def save(self, instance): + pass + + choices = backends.ServiceBackend.get_choices() + Route._meta.get_field('backend').choices = choices + backend = TestBackend.get_name() + + route = Route.objects.create(backend=backend, host=self.host, match='True') + operation = Operation(backend=TestBackend, instance=route, action='save') + self.assertEqual(1, len(Route.objects.get_for_operation(operation))) + + route = Route.objects.create(backend=backend, host=self.host1, + match='route.backend == "%s"' % TestBackend.get_name()) + self.assertEqual(2, len(Route.objects.get_for_operation(operation))) + + route = Route.objects.create(backend=backend, host=self.host2, + match='route.backend == "something else"') + self.assertEqual(2, len(Route.objects.get_for_operation(operation))) diff --git a/orchestra/contrib/orchestration/utils.py b/orchestra/contrib/orchestration/utils.py new file mode 100644 index 00000000..df59f8c9 --- /dev/null +++ b/orchestra/contrib/orchestration/utils.py @@ -0,0 +1,37 @@ +from orchestra.utils.sys import run, sshrun, join + + +def retrieve_state(servers): + uptimes = [] + pings = [] + for server in servers: + address = server.get_address() + ping = run('ping -c 1 -w 1 %s' % address, run_async=True) + pings.append(ping) + uptime = sshrun(address, 'uptime', persist=True, run_async=True, options={'ConnectTimeout': 1}) + uptimes.append(uptime) + + state = {} + for server, ping, uptime in zip(servers, pings, uptimes): + ping = join(ping, silent=True) + + try: + ping = ping.stdout.splitlines()[-1].decode() + except IndexError: + ping = '' + + if ping.startswith('rtt'): + ping = '%s ms' % ping.split('/')[4] + else: + ping = 'Offline' + + uptime = join(uptime, silent=True) + uptime_stderr = uptime.stderr.decode() + uptime = uptime.stdout.decode().split() + if uptime: + uptime = 'Up %s %s load %s %s %s' % (uptime[2], uptime[3], uptime[-3], uptime[-2], uptime[-1]) + else: + uptime = '%s' % uptime_stderr + state[server.pk] = (ping, uptime) + + return state diff --git a/orchestra/contrib/orchestration/widgets.py b/orchestra/contrib/orchestration/widgets.py new file mode 100644 index 00000000..576de4e8 --- /dev/null +++ b/orchestra/contrib/orchestration/widgets.py @@ -0,0 +1,24 @@ +import textwrap + +from orchestra.forms.widgets import DynamicHelpTextSelect + + +class RouteBackendSelect(DynamicHelpTextSelect): + """ Updates matches input field based on selected backend """ + def __init__(self, target, help_text, route_matches, *args, **kwargs): + kwargs['attrs'] = { + 'onfocus': "this.oldvalue = this.value;", + } + self.route_matches = route_matches + super(RouteBackendSelect, self).__init__(target, help_text, *args, **kwargs) + + def get_dynamic_help_text(self, target, help_text): + help_text = super(RouteBackendSelect, self).get_dynamic_help_text(target, help_text) + return help_text + textwrap.dedent("""\ + routematches = {route_matches}; + match = $("#id_match"); + if ( this.oldvalue == "" || match.value == routematches[this.oldvalue]) + match.value = routematches[this.options[this.selectedIndex].value]; + this.oldvalue = this.value; + """.format(route_matches=self.route_matches) + ) diff --git a/orchestra/contrib/orders/__init__.py b/orchestra/contrib/orders/__init__.py new file mode 100644 index 00000000..753c362c --- /dev/null +++ b/orchestra/contrib/orders/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.orders.apps.OrdersConfig' diff --git a/orchestra/contrib/orders/actions.py b/orchestra/contrib/orders/actions.py new file mode 100644 index 00000000..ba2244ec --- /dev/null +++ b/orchestra/contrib/orders/actions.py @@ -0,0 +1,174 @@ +from django.contrib import admin, messages +from django.urls import reverse +from django.db import transaction +from django.utils import timezone +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext_lazy as _ +from django.shortcuts import render + +from orchestra.admin.utils import change_url + +from .forms import BillSelectedOptionsForm, BillSelectConfirmationForm, BillSelectRelatedForm + + +class BillSelectedOrders(object): + """ Form wizard for billing orders admin action """ + short_description = _("Bill selected orders") + verbose_name = _("Bill") + template = 'admin/orders/order/bill_selected_options.html' + __name__ = 'bill_selected_orders' + + def __call__(self, modeladmin, request, queryset): + """ make this monster behave like a function """ + self.modeladmin = modeladmin + self.queryset = queryset + opts = modeladmin.model._meta + app_label = opts.app_label + self.context = { + 'opts': opts, + 'app_label': app_label, + 'queryset': queryset, + 'action_checkbox_name': admin.helpers.ACTION_CHECKBOX_NAME, + } + ret = self.set_options(request) + del(self.queryset) + del(self.context) + return ret + + def set_options(self, request): + form = BillSelectedOptionsForm() + if request.POST.get('step'): + form = BillSelectedOptionsForm(request.POST) + if form.is_valid(): + self.options = dict( + billing_point=form.cleaned_data['billing_point'], + fixed_point=form.cleaned_data['fixed_point'], + proforma=form.cleaned_data['proforma'], + new_open=form.cleaned_data['new_open'], + ) + if int(request.POST.get('step')) != 3: + return self.select_related(request) + else: + return self.confirmation(request) + self.context.update({ + 'title': _("Options for billing selected orders, step 1 / 3"), + 'step': 1, + 'form': form, + }) + return render(request, self.template, self.context) + + def select_related(self, request): + # TODO use changelist ? + related = self.queryset.get_related().select_related('account', 'service') + if not related: + return self.confirmation(request) + self.options['related_queryset'] = related + form = BillSelectRelatedForm(initial=self.options) + if int(request.POST.get('step')) >= 2: + form = BillSelectRelatedForm(request.POST, initial=self.options) + if form.is_valid(): + select_related = form.cleaned_data['selected_related'] + self.queryset = self.queryset | select_related + return self.confirmation(request) + self.context.update({ + 'title': _("Select related order for billing, step 2 / 3"), + 'step': 2, + 'form': form, + }) + return render(request, self.template, self.context) + + @transaction.atomic + def confirmation(self, request): + form = BillSelectConfirmationForm(initial=self.options) + if int(request.POST.get('step')) >= 3: + bills = self.queryset.bill(commit=True, **self.options) + for order in self.queryset: + self.modeladmin.log_change(request, order, _("Billed")) + if not bills: + msg = _("Selected orders do not have pending billing") + self.modeladmin.message_user(request, msg, messages.WARNING) + else: + num = len(bills) + if num == 1: + url = change_url(bills[0]) + else: + url = reverse('admin:bills_bill_changelist') + ids = ','.join([str(b.id) for b in bills]) + url += '?id__in=%s' % ids + msg = ngettext( + 'One bill has been created.', + '{num} bills have been created.', + num).format(url=url, num=num) + msg = mark_safe(msg) + self.modeladmin.message_user(request, msg, messages.INFO) + return + bills = self.queryset.bill(commit=False, **self.options) + bills_with_total = [] + for account, lines in bills: + total = 0 + for line in lines: + discount = sum([discount.total for discount in line.discounts]) + total += line.subtotal + discount + bills_with_total.append((account, total, lines)) + self.context.update({ + 'title': _("Confirmation for billing selected orders"), + 'step': 3, + 'form': form, + 'bills': sorted(bills_with_total, key=lambda i: -i[1]), + }) + return render(request, self.template, self.context) + + +@transaction.atomic +def mark_as_ignored(modeladmin, request, queryset): + """ Mark orders as ignored """ + for order in queryset: + order.mark_as_ignored() + modeladmin.log_change(request, order, 'Marked as ignored') + num = len(queryset) + msg = ngettext( + _("Selected order has been marked as ignored."), + _("%i selected orders have been marked as ignored.") % num, + num) + modeladmin.message_user(request, msg) + + +@transaction.atomic +def mark_as_not_ignored(modeladmin, request, queryset): + """ Mark orders as ignored """ + for order in queryset: + order.mark_as_not_ignored() + modeladmin.log_change(request, order, 'Marked as not ignored') + num = len(queryset) + msg = ngettext( + _("Selected order has been marked as not ignored."), + _("%i selected orders have been marked as not ignored.") % num, + num) + modeladmin.message_user(request, msg) + + +def report(modeladmin, request, queryset): + services = {} + totals = [0, 0, None, 0] + now = timezone.now().date() + for order in queryset.select_related('service'): + name = order.service.description + active, cancelled = (1, 0) if not order.cancelled_on or order.cancelled_on > now else (0, 1) + try: + info = services[name] + except KeyError: + nominal_price = order.service.nominal_price + info = [active, cancelled, nominal_price, 1] + services[name] = info + else: + info[0] += active + info[1] += cancelled + info[3] += 1 + totals[0] += active + totals[1] += cancelled + totals[3] += 1 + context = { + 'services': sorted(services.items(), key=lambda n: -n[1][0]), + 'totals': totals, + } + return render(request, 'admin/orders/order/report.html', context) diff --git a/orchestra/contrib/orders/admin.py b/orchestra/contrib/orders/admin.py new file mode 100644 index 00000000..3c3086b8 --- /dev/null +++ b/orchestra/contrib/orders/admin.py @@ -0,0 +1,207 @@ +from datetime import datetime +from django import forms +from django.contrib import admin +from django.urls import reverse, NoReverseMatch +from django.db.models import Prefetch +from django.utils import timezone +from django.utils.html import escape, format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import admin_link, admin_date, change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.utils.humanize import naturaldate + +from .actions import BillSelectedOrders, mark_as_ignored, mark_as_not_ignored, report +from .filters import IgnoreOrderListFilter, ActiveOrderListFilter, BilledOrderListFilter +from .models import Order, MetricStorage + + +class MetricStorageInline(admin.TabularInline): + model = MetricStorage + readonly_fields = ('value', 'created_on', 'updated_on') + extra = 0 + + def has_add_permission(self, request, obj=None): + return False + + def get_fieldsets(self, request, obj=None): + if obj: + url = reverse('admin:orders_metricstorage_changelist') + url += '?order=%i' % obj.pk + title = _('Metric storage, last 10 entries, (See all)') + self.verbose_name_plural = mark_safe(title % url) + return super(MetricStorageInline, self).get_fieldsets(request, obj) + + def get_queryset(self, request): + qs = super(MetricStorageInline, self).get_queryset(request) + change_view = bool(self.parent_object and self.parent_object.pk) + if change_view: + qs = qs.order_by('-id') + parent_id = self.parent_object.pk + try: + tenth_id = qs.filter(order_id=parent_id).values_list('id', flat=True)[9] + except IndexError: + pass + else: + return qs.filter(pk__gte=tenth_id) + return qs + + +class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'display_description', 'service_link', 'account_link', 'content_object_link', + 'display_registered_on', 'display_billed_until', 'display_cancelled_on', + 'display_metric' + ) + list_filter = ( + ActiveOrderListFilter, IgnoreOrderListFilter, BilledOrderListFilter, 'account__type', + 'service', + ) + default_changelist_filters = ( + ('ignore', '0'), + ) + actions = ( + BillSelectedOrders(), mark_as_ignored, mark_as_not_ignored, report, list_accounts + ) + change_view_actions = (BillSelectedOrders(), mark_as_ignored, mark_as_not_ignored) + date_hierarchy = 'registered_on' + inlines = (MetricStorageInline,) + add_inlines = () + search_fields = ('account__username', 'content_object_repr', 'description',) + list_prefetch_related = ( + 'content_object', + Prefetch('metrics', queryset=MetricStorage.objects.order_by('-id')), + ) + list_select_related = ('account', 'service') + add_fieldsets = ( + (None, { + 'fields': ('account', 'service') + }), + (_("Object"), { + 'fields': ('content_type', 'object_id',), + }), + (_("State"), { + 'fields': ('registered_on', 'cancelled_on', 'billed_on', 'billed_metric', + 'billed_until' ) + }), + (None, { + 'fields': ('description', 'ignore',), + }), + ) + fieldsets = ( + (None, { + 'fields': ('account_link', 'service_link', 'content_object_link'), + }), + (_("State"), { + 'fields': ('registered_on', 'cancelled_on', 'billed_on', 'billed_metric', + 'billed_until' ) + }), + (None, { + 'fields': ('description', 'ignore', 'bills_links'), + }), + ) + readonly_fields = ( + 'content_object_repr', 'content_object_link', 'bills_links', 'account_link', + 'service_link' + ) + + service_link = admin_link('service') + display_registered_on = admin_date('registered_on') + display_cancelled_on = admin_date('cancelled_on') + + def display_description(self, order): + return format_html(order.description[:64]) + display_description.short_description = _("Description") + display_description.admin_order_field = 'description' + + def content_object_link(self, order): + if order.content_object: + try: + url = change_url(order.content_object) + except NoReverseMatch: + # Does not has admin + return order.content_object_repr + description = str(order.content_object) + return format_html('{description}', + url=url, description=description) + return order.content_object_repr + content_object_link.short_description = _("Content object") + content_object_link.admin_order_field = 'content_object_repr' + + @mark_safe + def bills_links(self, order): + bills = [] + make_link = admin_link() + for line in order.lines.select_related('bill').distinct('bill'): + bills.append(make_link(line.bill)) + return '
    '.join(bills) + bills_links.short_description = _("Bills") + + def display_billed_until(self, order): + billed_until = order.billed_until + red = False + human = escape(naturaldate(billed_until)) + if billed_until: + if order.cancelled_on and order.cancelled_on <= billed_until: + pass + elif order.service.billing_period == order.service.NEVER: + human = _("Forever") + elif order.service.payment_style == order.service.POSTPAY: + boundary = order.service.handler.get_billing_point(order) + if billed_until < boundary: + red = True + elif billed_until < timezone.now().date(): + red = True + color = mark_safe('style="color:red;"') if red else '' + return format_html( + '{human}', + raw=escape(str(billed_until)), color=color, human=human, + ) + display_billed_until.short_description = _("billed until") + display_billed_until.admin_order_field = 'billed_until' + + def display_metric(self, order): + """ + dispalys latest metric value, don't uses latest() because not loosing prefetch_related + """ + try: + metric = order.metrics.all()[0] + except IndexError: + return '' + return metric.value + display_metric.short_description = _("Metric") + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Make value input widget bigger """ + if db_field.name == 'description': + kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 2}) + return super().formfield_for_dbfield(db_field, **kwargs) + +# def get_changelist(self, request, **kwargs): +# ChangeList = super(OrderAdmin, self).get_changelist(request, **kwargs) +# class OrderFilterChangeList(ChangeList): +# def get_filters(self, request): +# filters = super(OrderFilterChangeList, self).get_filters(request) +# tail = [] +# filters_copy = [] +# for list_filter in filters[0]: +# if getattr(list_filter, 'apply_last', False): +# tail.append(list_filter) +# else: +# filters_copy.append(list_filter) +# filters = ((filters_copy+tail),) + filters[1:] +# return filters +# return OrderFilterChangeList + + +class MetricStorageAdmin(admin.ModelAdmin): + list_display = ('order', 'value', 'created_on', 'updated_on') + list_filter = ('order__service',) + raw_id_fields = ('order',) + + +admin.site.register(Order, OrderAdmin) +admin.site.register(MetricStorage, MetricStorageAdmin) diff --git a/orchestra/contrib/orders/api.py b/orchestra/contrib/orders/api.py new file mode 100644 index 00000000..922632eb --- /dev/null +++ b/orchestra/contrib/orders/api.py @@ -0,0 +1,15 @@ +from rest_framework import viewsets + +from orchestra.api import router +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import Order +from .serializers import OrderSerializer + + +class OrderViewSet(AccountApiMixin, viewsets.ModelViewSet): + queryset = Order.objects.all() + serializer_class = OrderSerializer + + +router.register(r'orders', OrderViewSet) diff --git a/orchestra/contrib/orders/apps.py b/orchestra/contrib/orders/apps.py new file mode 100644 index 00000000..1d526985 --- /dev/null +++ b/orchestra/contrib/orders/apps.py @@ -0,0 +1,13 @@ +from django.apps import AppConfig + +from orchestra.core import accounts + + +class OrdersConfig(AppConfig): + name = 'orchestra.contrib.orders' + verbose_name = 'Orders' + + def ready(self): + from .models import Order + accounts.register(Order, icon='basket.png', search=False) + from . import signals diff --git a/orchestra/contrib/orders/billing.py b/orchestra/contrib/orders/billing.py new file mode 100644 index 00000000..a6daee33 --- /dev/null +++ b/orchestra/contrib/orders/billing.py @@ -0,0 +1,96 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.bills.models import Invoice, Fee, ProForma + + +class BillsBackend(object): + def create_bills(self, account, lines, **options): + bill = None + ant_bill = None + bills = [] + create_new = options.get('new_open', False) + proforma = options.get('proforma', False) + for line in lines: + quantity = line.metric*line.size + if quantity == 0: + continue + service = line.order.service + # Create bill if needed + if proforma: + if ant_bill is None: + if create_new: + bill = ProForma.objects.create(account=account) + else: + bill = ProForma.objects.filter(account=account, is_open=True).last() + if bill: + bill.updated() + else: + bill = ProForma.objects.create(account=account, is_open=True) + bills.append(bill) + else: + bill = ant_bill + ant_bill = bill + elif service.is_fee: + bill = Fee.objects.create(account=account) + bills.append(bill) + else: + if ant_bill is None: + if create_new: + bill = Invoice.objects.create(account=account) + else: + bill = Invoice.objects.filter(account=account, is_open=True).last() + if bill: + bill.updated() + else: + bill = Invoice.objects.create(account=account, is_open=True) + bills.append(bill) + else: + bill = ant_bill + ant_bill = bill + # Create bill line + billine = bill.lines.create( + rate=service.nominal_price, + quantity=line.metric*line.size, + verbose_quantity=self.get_verbose_quantity(line), + subtotal=line.subtotal, + tax=service.tax, + description=self.get_line_description(line), + start_on=line.ini, + end_on=line.end if service.billing_period != service.NEVER else None, + order=line.order, + order_billed_on=line.order.old_billed_on, + order_billed_until=line.order.old_billed_until + ) + self.create_sublines(billine, line.discounts) + return bills + +# def format_period(self, ini, end): +# ini = ini.strftime("%b, %Y") +# end = (end-datetime.timedelta(seconds=1)).strftime("%b, %Y") +# if ini == end: +# return ini +# return _("{ini} to {end}").format(ini=ini, end=end) + + def get_line_description(self, line): + service = line.order.service + description = line.order.description + return description + + def get_verbose_quantity(self, line): + metric = format(line.metric, '.2f').rstrip('0').rstrip('.') + metric = metric.strip('0').strip('.') + size = format(line.size, '.2f').rstrip('0').rstrip('.') + size = size.strip('0').strip('.') + if metric == '1': + return size + if size == '1': + return metric + return "%s×%s" % (metric, size) + + def create_sublines(self, line, discounts): + for discount in discounts: + line.sublines.create( + description=_("Discount per %s") % discount.type.lower(), + total=discount.total, + type=discount.type, + ) diff --git a/orchestra/contrib/orders/filters.py b/orchestra/contrib/orders/filters.py new file mode 100644 index 00000000..4693529f --- /dev/null +++ b/orchestra/contrib/orders/filters.py @@ -0,0 +1,131 @@ +from datetime import timedelta + +from django.apps import apps +from django.contrib.admin import SimpleListFilter +from django.db.models import Q, Prefetch, F +from django.utils import timezone +from django.utils.encoding import force_str +from django.utils.translation import gettext_lazy as _ + +from . import settings +from .models import MetricStorage + + +class ActiveOrderListFilter(SimpleListFilter): + """ Filter tickets by created_by according to request.user """ + title = _("is active") + parameter_name = 'is_active' + + def lookups(self, request, model_admin): + return ( + ('True', _("Active")), + ('False', _("Inactive")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.active() + elif self.value() == 'False': + return queryset.inactive() + return queryset + + +class BilledOrderListFilter(SimpleListFilter): + """ Filter tickets by created_by according to request.user """ + title = _("billed") + parameter_name = 'billed' +# apply_last = True + + def lookups(self, request, model_admin): + return ( + ('yes', _("Billed")), + ('no', _("Not billed")), + ('pending', _("Pending (re-evaluate metric)")), + ('not_pending', _("Not pending (re-evaluate metric)")), + ) + + def get_pending_metric_pks(self, queryset): + mindelta = timedelta(days=2) # TODO + metric_pks = [] + prefetch_valid_metrics = Prefetch('metrics', to_attr='valid_metrics', + queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'), + created_on__lte=(F('updated_on')-mindelta)).exclude(value=0) + ) + metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True) + for order in metric_queryset.prefetch_related(prefetch_valid_metrics): + for metric in order.valid_metrics: + if metric.created_on <= order.billed_on: + raise ValueError("This value should already be filtered on the prefetch query.") + if metric.value > order.billed_metric: + metric_pks.append(order.pk) + break + return metric_pks + + def filter_pending(self, queryset, reverse=False): + now = timezone.now() + Service = apps.get_model(settings.ORDERS_SERVICE_MODEL) + ignore_qs = Q() + for order in queryset.distinct('service_id').only('service'): + service = order.service + delta = service.handler.get_ignore_delta() + if delta is not None: + ignore_qs = ignore_qs | Q(service_id=service.id, registered_on__gt=now-delta) + ignore_qs = queryset.exclude(ignore_qs) + pending_qs = Q( + Q(pk__in=self.get_pending_metric_pks(ignore_qs)) | + Q(billed_until__isnull=True) | Q(~Q(service__billing_period=Service.NEVER) & + Q(billed_until__lte=now)) + ) + if reverse: + return queryset.exclude(pending_qs) + else: + return ignore_qs.filter(pending_qs) + + def queryset(self, request, queryset): + now = timezone.now() + if self.value() == 'yes': + return queryset.filter(billed_until__isnull=False, billed_until__gte=now) + elif self.value() == 'no': + return queryset.exclude(billed_until__isnull=False, billed_until__gte=now) + elif self.value() == 'pending': + return self.filter_pending(queryset) + elif self.value() == 'not_pending': + return self.filter_pending(queryset, reverse=True) + return queryset + + +class IgnoreOrderListFilter(SimpleListFilter): + """ Filter Nodes by group according to request.user """ + title = _("Ignore") + parameter_name = 'ignore' + + def lookups(self, request, model_admin): + return ( + ('0', _("Not ignored")), + ('1', _("Ignored")), + ('2', _("All")), + + ) + + def queryset(self, request, queryset): + if self.value() == '0': + return queryset.filter(ignore=False) + elif self.value() == '1': + return queryset.filter(ignore=True) + return queryset + + def choices(self, cl): + """ Enable default selection different than All """ + for lookup, title in self.lookup_choices: + title = title._proxy____args[0] + selected = self.value() == force_str(lookup) + if not selected and title == "Not ignored" and self.value() is None: + selected = True + # end of workaround + yield { + 'selected': selected, + 'query_string': cl.get_query_string({ + self.parameter_name: lookup, + }, []), + 'display': title, + } diff --git a/orchestra/contrib/orders/forms.py b/orchestra/contrib/orders/forms.py new file mode 100644 index 00000000..0dd9a0ab --- /dev/null +++ b/orchestra/contrib/orders/forms.py @@ -0,0 +1,70 @@ +from django import forms +from django.contrib.admin import widgets +from django.utils import timezone +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin.forms import AdminFormMixin +from orchestra.admin.utils import change_url + +from .models import Order + + +class BillSelectedOptionsForm(AdminFormMixin, forms.Form): + billing_point = forms.DateField(initial=timezone.now, + label=_("Billing point"), widget=widgets.AdminDateWidget, + help_text=_("Date you want to bill selected orders")) + fixed_point = forms.BooleanField(initial=False, required=False, + label=_("Fixed point"), + help_text=_("Deisgnates whether you want the billing point to be an " + "exact date, or adapt it to the billing period.")) + proforma = forms.BooleanField(initial=False, required=False, + label=_("Pro-forma (billing simulation)"), + help_text=_("Creates a Pro Forma instead of billing the orders.")) + new_open = forms.BooleanField(initial=False, required=False, + label=_("Create a new open bill"), + help_text=_("Deisgnates whether you want to put this orders on a new " + "open bill, or allow to reuse an existing one.")) + + +def selected_related_choices(queryset): + for order in queryset: + verbose = '{description} ' + verbose += '' + if order.ignore: + verbose += ' (ignored)' + verbose = verbose.format( + order_url=change_url(order), description=order.description, + account_url=change_url(order.account), account=str(order.account) + ) + yield (order.pk, mark_safe(verbose)) + + +class BillSelectRelatedForm(AdminFormMixin, forms.Form): + # This doesn't work well with reordering after billing +# pricing_with_all = forms.BooleanField(label=_("Do pricing with all orders"), +# initial=False, required=False, help_text=_("The price may vary " +# "depending on the billed orders. This options designates whether " +# "all existing orders will be used for price computation or not.")) + select_all = forms.BooleanField(label=_("Select all"), required=False) + selected_related = forms.ModelMultipleChoiceField(label=_("Related orders"), + queryset=Order.objects.none(), widget=forms.CheckboxSelectMultiple, + required=False) + billing_point = forms.DateField(widget=forms.HiddenInput()) + fixed_point = forms.BooleanField(widget=forms.HiddenInput(), required=False) + proforma = forms.BooleanField(widget=forms.HiddenInput(), required=False) + new_open = forms.BooleanField(widget=forms.HiddenInput(), required=False) + + def __init__(self, *args, **kwargs): + super(BillSelectRelatedForm, self).__init__(*args, **kwargs) + queryset = kwargs['initial'].get('related_queryset', None) + if queryset: + self.fields['selected_related'].queryset = queryset + self.fields['selected_related'].choices = selected_related_choices(queryset) + + +class BillSelectConfirmationForm(AdminFormMixin, forms.Form): + billing_point = forms.DateField(widget=forms.HiddenInput()) + fixed_point = forms.BooleanField(widget=forms.HiddenInput(), required=False) + proforma = forms.BooleanField(widget=forms.HiddenInput(), required=False) + new_open = forms.BooleanField(widget=forms.HiddenInput(), required=False) diff --git a/orchestra/contrib/orders/helpers.py b/orchestra/contrib/orders/helpers.py new file mode 100644 index 00000000..ce0d1917 --- /dev/null +++ b/orchestra/contrib/orders/helpers.py @@ -0,0 +1,39 @@ +from django.core.exceptions import ObjectDoesNotExist + +from orchestra.core import services + + +def get_related_object(origin, max_depth=2): + """ + Introspects origin object and return the first related service object + + WARNING this is NOT an exhaustive search but a compromise between cost and + flexibility. A more comprehensive approach may be considered if + a use-case calls for it. + """ + def related_iterator(node): + for field in node._meta.private_fields: + if hasattr(field, 'ct_field'): + yield getattr(node, field.name) + for field in node._meta.fields: + if field.remote_field: + try: + yield getattr(node, field.name) + except ObjectDoesNotExist: + pass + + # BFS model relation transversal + queue = [[origin]] + while queue: + models = queue.pop(0) + if len(models) > max_depth: + return None + node = models[-1] + if len(models) > 1: + if type(node) in services: + return node + for related in related_iterator(node): + if related and related not in models: + new_models = list(models) + new_models.append(related) + queue.append(new_models) diff --git a/orchestra/contrib/orders/models.py b/orchestra/contrib/orders/models.py new file mode 100644 index 00000000..db6febc8 --- /dev/null +++ b/orchestra/contrib/orders/models.py @@ -0,0 +1,310 @@ +import datetime +import decimal +import logging + +from django.db import models +from django.db.models import F, Q, Sum +from django.apps import apps +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from orchestra.models import queryset +from orchestra.utils.python import import_class + +from . import settings + + +logger = logging.getLogger(__name__) + + +class OrderQuerySet(models.QuerySet): + group_by = queryset.group_by + + def bill(self, **options): + bills = [] + bill_backend = Order.get_bill_backend() + qs = self.select_related('account', 'service') + commit = options.get('commit', True) + for account, services in qs.group_by('account', 'service').items(): + bill_lines = [] + for service, orders in services.items(): + for order in orders: + # Saved for undoing support + order.old_billed_on = order.billed_on + order.old_billed_until = order.billed_until + lines = service.handler.generate_bill_lines(orders, account, **options) + bill_lines.extend(lines) + # TODO make this consistent always returning the same fucking types + if commit: + bills += bill_backend.create_bills(account, bill_lines, **options) + else: + bills += [(account, bill_lines)] + # TODO remove if commit and always return unique elemenets (set()) when the other todo is fixed + if commit: + return list(set(bills)) + return bills + + def givers(self, ini, end): + return self.cancelled_and_billed().filter(billed_until__gt=ini, registered_on__lt=end) + + def cancelled_and_billed(self, exclude=False): + qs = dict(cancelled_on__isnull=False, billed_until__isnull=False, + cancelled_on__lte=F('billed_until')) + if exclude: + return self.exclude(**qs) + return self.filter(**qs) + + def get_related(self, **options): + """ returns related orders that could have a pricing effect """ + Service = apps.get_model(settings.ORDERS_SERVICE_MODEL) + conflictive = self.filter(service__metric='') + conflictive = conflictive.exclude(service__billing_period=Service.NEVER) + # Exclude rates null or all rates with quantity 0 + conflictive = conflictive.annotate(quantity_sum=Sum('service__rates__quantity')) + conflictive = conflictive.exclude(quantity_sum=0).select_related('service').distinct() + qs = Q() + for account_id, services in conflictive.group_by('account_id', 'service').items(): + for service, orders in services.items(): + ini = datetime.date.max + end = datetime.date.min + bp = None + for order in orders: + bp = service.handler.get_billing_point(order, **options) + end = max(end, bp) + ini = min(ini, order.billed_until or order.registered_on) + qs |= Q( + Q(service=service, account=account_id, registered_on__lt=end) & Q( + Q(billed_until__isnull=True) | Q(billed_until__lt=end) + ) & Q( + Q(cancelled_on__isnull=True) | Q(cancelled_on__gt=ini) + ) + ) + if not qs: + return self.model.objects.none() + ids = self.values_list('id', flat=True) + return self.model.objects.filter(qs).exclude(id__in=ids) + + def pricing_orders(self, ini, end): + return self.filter(billed_until__isnull=False, billed_until__gt=ini, + registered_on__lt=end) + + def by_object(self, obj, **kwargs): + ct = ContentType.objects.get_for_model(obj) + return self.filter(object_id=obj.pk, content_type=ct, **kwargs) + + def active(self, **kwargs): + """ return active orders """ + return self.filter( + Q(cancelled_on__isnull=True) | Q(cancelled_on__gt=timezone.now()) + ).filter(**kwargs) + + def inactive(self, **kwargs): + """ return inactive orders """ + return self.filter(cancelled_on__lte=timezone.now(), **kwargs) + + def update_by_instance(self, instance, service=None, commit=True): + updates = [] + if service is None: + Service = apps.get_model(settings.ORDERS_SERVICE_MODEL) + services = Service.objects.filter_by_instance(instance) + else: + services = [service] + for service in services: + orders = Order.objects.by_object(instance, service=service) + orders = orders.select_related('service').active() + if service.handler.matches(instance): + if not orders: + account_id = getattr(instance, 'account_id', instance.pk) + if account_id is None: + # New account workaround -> user.account_id == None + continue + ignore = service.handler.get_ignore(instance) + order = self.model( + content_object=instance, + content_object_repr=str(instance), + service=service, + account_id=account_id, + ignore=ignore) + if commit: + order.save() + updates.append((order, 'created')) + logger.info("CREATED new order id: {id}".format(id=order.id)) + else: + if len(orders) > 1: + raise ValueError("A single active order was expected.") + order = orders[0] + updates.append((order, 'updated')) + if commit: + order.update() + elif orders: + if len(orders) > 1: + raise ValueError("A single active order was expected.") + order = orders[0] + order.cancel(commit=commit) + logger.info("CANCELLED order id: {id}".format(id=order.id)) + updates.append((order, 'cancelled')) + return updates + + +class Order(models.Model): + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='orders') + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + object_id = models.PositiveIntegerField(null=True) + service = models.ForeignKey(settings.ORDERS_SERVICE_MODEL, on_delete=models.PROTECT, + verbose_name=_("service"), related_name='orders') + registered_on = models.DateField(_("registered"), default=timezone.now, db_index=True) + cancelled_on = models.DateField(_("cancelled"), null=True, blank=True) + billed_on = models.DateField(_("billed"), null=True, blank=True) + billed_metric = models.DecimalField(_("billed metric"), max_digits=16, decimal_places=2, + null=True, blank=True) + billed_until = models.DateField(_("billed until"), null=True, blank=True) + ignore = models.BooleanField(_("ignore"), default=False) + description = models.TextField(_("description"), blank=True) + content_object_repr = models.CharField(_("content object representation"), max_length=256, + editable=False, help_text=_("Used for searches.")) + + content_object = GenericForeignKey() + objects = OrderQuerySet.as_manager() + + class Meta: + get_latest_by = 'id' + index_together = ( + ('content_type', 'object_id'), + ) + + def __str__(self): + return str(self.service) + + @classmethod + def get_bill_backend(cls): + return import_class(settings.ORDERS_BILLING_BACKEND)() + + def clean(self): + if self.billed_on and self.billed_on < self.registered_on: + raise ValidationError(_("Billed date can not be earlier than registered on.")) + if self.billed_until and not self.billed_on: + raise ValidationError(_("Billed on is missing while billed until is being provided.")) + + def update(self): + instance = self.content_object + if instance is None: + return + handler = self.service.handler + metric = '' + if handler.metric: + metric = handler.get_metric(instance) + if metric is not None: + MetricStorage.objects.store(self, metric) + metric = ', metric:{}'.format(metric) + description = handler.get_order_description(instance) + logger.info("UPDATED order id:{id}, description:{description}{metric}".format( + id=self.id, description=description, metric=metric).encode('ascii', 'replace') + ) + update_fields = [] + if self.description != description: + self.description = description + update_fields.append('description') + content_object_repr = str(instance) + if self.content_object_repr != content_object_repr: + self.content_object_repr = content_object_repr + update_fields.append('content_object_repr') + if update_fields: + self.save(update_fields=update_fields) + + def cancel(self, commit=True): + self.cancelled_on = timezone.now() + self.ignore = self.service.handler.get_order_ignore(self) + if commit: + self.save(update_fields=['cancelled_on', 'ignore']) + logger.info("CANCELLED order id: {id}".format(id=self.id)) + + def mark_as_ignored(self): + self.ignore = True + self.save(update_fields=['ignore']) + + def mark_as_not_ignored(self): + self.ignore = False + self.save(update_fields=['ignore']) + + def get_metric(self, *args, **kwargs): + if kwargs.pop('changes', False): + ini, end = args + result = [] + prev = None + for metric in self.metrics.filter(created_on__lt=end).order_by('id'): + created = metric.created_on + if created > ini: + if prev is None: + raise ValueError("Metric storage information for order %i is inconsistent." % self.id) + cini = prev.created_on + if not result: + cini = ini + result.append((cini, created, prev.value)) + prev = metric + if created < end: + result.append((created, end, metric.value)) + return result + if kwargs: + raise AttributeError + if len(args) == 2: + # Slot + ini, end = args + metrics = self.metrics.filter(created_on__lt=end, updated_on__gte=ini) + elif len(args) == 1: + # On effect on date + date = args[0] + date = datetime.date(year=date.year, month=date.month, day=date.day) + date += datetime.timedelta(days=1) + metrics = self.metrics.filter(created_on__lte=date) + elif not args: + return self.metrics.latest('updated_on').value + else: + raise AttributeError + try: + return metrics.latest('updated_on').value + except MetricStorage.DoesNotExist: + return decimal.Decimal(0) + + +class MetricStorageQuerySet(models.QuerySet): + def store(self, order, value): + now = timezone.now() + try: + last = self.filter(order=order).latest() + except self.model.DoesNotExist: + self.create(order=order, value=value, updated_on=now) + else: + # Metric storage has per-day granularity (last value of the day is what counts) + if last.created_on == now.date(): + last.value = value + last.updated_on = now + last.save() + else: + error = decimal.Decimal(str(settings.ORDERS_METRIC_ERROR)) + if (value > last.value+error or value < last.value-error) or (value == 0 and last.value > 0): + self.create(order=order, value=value, updated_on=now) + else: + last.updated_on = now + last.save(update_fields=['updated_on']) + + +class MetricStorage(models.Model): + """ Stores metric state for future billing """ + order = models.ForeignKey(Order, on_delete=models.CASCADE, + verbose_name=_("order"), related_name='metrics') + value = models.DecimalField(_("value"), max_digits=16, decimal_places=2) + created_on = models.DateField(_("created"), auto_now_add=True, editable=True) + # TODO time field? + updated_on = models.DateTimeField(_("updated")) + + objects = MetricStorageQuerySet.as_manager() + + class Meta: + get_latest_by = 'id' + + def __str__(self): + return str(self.order) diff --git a/orchestra/contrib/orders/serializers.py b/orchestra/contrib/orders/serializers.py new file mode 100644 index 00000000..ea30240d --- /dev/null +++ b/orchestra/contrib/orders/serializers.py @@ -0,0 +1,14 @@ +from rest_framework import serializers + +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .models import Order + + +class OrderSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + class Meta: + model = Order + fields = ( + 'url', 'id', 'registered_on', 'cancelled_on', 'billed_on', 'billed_until', + 'description' + ) diff --git a/orchestra/contrib/orders/settings.py b/orchestra/contrib/orders/settings.py new file mode 100644 index 00000000..2dbed0b5 --- /dev/null +++ b/orchestra/contrib/orders/settings.py @@ -0,0 +1,46 @@ +from orchestra.contrib.settings import Setting + + +ORDERS_BILLING_BACKEND = Setting('ORDERS_BILLING_BACKEND', + 'orchestra.contrib.orders.billing.BillsBackend', + validators=[Setting.validate_import_class], + help_text="Pluggable backend for bill generation.", +) + + +ORDERS_SERVICE_MODEL = Setting('ORDERS_SERVICE_MODEL', + 'services.Service', + validators=[Setting.validate_model_label], + help_text="Pluggable service class.", +) + + +ORDERS_EXCLUDED_APPS = Setting('ORDERS_EXCLUDED_APPS', + ( + 'orders', + 'admin', + 'contenttypes', + 'auth', + 'migrations', + 'sessions', + 'orchestration', + 'bills', + 'services', + 'mailer', + 'issues', + ), + help_text="Prevent inspecting these apps for service accounting." +) + + +ORDERS_METRIC_ERROR = Setting('ORDERS_METRIC_ERROR', + 0.05, + help_text=("Only account for significative changes.
    " + "metric_storage new value: lastvalue*(1+threshold) > currentvalue or lastvalue*threshold < currentvalue."), +) + + +ORDERS_BILLED_METRIC_CLEANUP_DAYS = Setting('ORDERS_BILLED_METRIC_CLEANUP_DAYS', + 40, + help_text=("Number of days after a billed stored metric is deleted."), +) diff --git a/orchestra/contrib/orders/signals.py b/orchestra/contrib/orders/signals.py new file mode 100644 index 00000000..1778a3da --- /dev/null +++ b/orchestra/contrib/orders/signals.py @@ -0,0 +1,39 @@ +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver + +from orchestra.core import services + +from . import helpers, settings +from .models import Order + + +# TODO perhas use cache = caches.get_request_cache() to cache an account delete and don't processes get_related_objects() if the case +# FIXME https://code.djangoproject.com/ticket/24576 +# TODO build a cache hash table {model: related, model: None} +@receiver(post_delete, dispatch_uid="orders.cancel_orders") +def cancel_orders(sender, **kwargs): + if sender._meta.app_label not in settings.ORDERS_EXCLUDED_APPS: + instance = kwargs['instance'] + # Account delete will delete all related orders, no need to maintain order consistency + if isinstance(instance, Order.account.field.related_model): + return + if type(instance) in services: + for order in Order.objects.by_object(instance).active(): + order.cancel() + elif not hasattr(instance, 'account'): + # FIXME Indeterminate behaviour + related = helpers.get_related_object(instance) + if related and related != instance: + type(related).objects.get(pk=related.pk) + + +@receiver(post_save, dispatch_uid="orders.update_orders") +def update_orders(sender, **kwargs): + if sender._meta.app_label not in settings.ORDERS_EXCLUDED_APPS: + instance = kwargs['instance'] + if type(instance) in services: + Order.objects.update_by_instance(instance) + elif not hasattr(instance, 'account'): + related = helpers.get_related_object(instance) + if related and related != instance: + Order.objects.update_by_instance(related) diff --git a/orchestra/contrib/orders/tasks.py b/orchestra/contrib/orders/tasks.py new file mode 100644 index 00000000..6089e652 --- /dev/null +++ b/orchestra/contrib/orders/tasks.py @@ -0,0 +1,50 @@ +import datetime + +from celery.task.schedules import crontab +from django.apps import apps + +from orchestra.contrib.tasks import periodic_task + +from . import settings + + +@periodic_task(run_every=crontab(hour=4, minute=30), name='orders.cleanup_metrics') +def cleanup_metrics(): + from .models import MetricStorage, Order + Service = apps.get_model(settings.ORDERS_SERVICE_MODEL) + + # General cleaning: order.billed_on-delta + general = 0 + delta = datetime.timedelta(days=settings.ORDERS_BILLED_METRIC_CLEANUP_DAYS) + for order in Order.objects.filter(billed_on__isnull=False): + epoch = order.billed_on-delta + try: + latest = order.metrics.filter(updated_on__lt=epoch).latest('updated_on') + except MetricStorage.DoesNotExist: + pass + else: + general += order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).count() + order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).only('id').delete() + + # Reduce monthly metrics to latest + monthly = 0 + monthly_services = Service.objects.exclude(metric='').filter( + billing_period=Service.MONTHLY, pricing_period=Service.BILLING_PERIOD + ) + for service in monthly_services: + for order in Order.objects.filter(service=service): + dates = order.metrics.values_list('created_on', flat=True) + months = set((date.year, date.month) for date in dates) + for year, month in months: + metrics = order.metrics.filter( + created_on__year=year, created_on__month=month, + updated_on__year=year, updated_on__month=month) + try: + latest = metrics.latest('updated_on') + except MetricStorage.DoesNotExist: + pass + else: + monthly += metrics.exclude(pk=latest.pk).count() + metrics.exclude(pk=latest.pk).only('id').delete() + + return (general, monthly) diff --git a/orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html b/orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html new file mode 100644 index 00000000..6dd023f6 --- /dev/null +++ b/orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html @@ -0,0 +1,98 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n static admin_urls utils orders %} + +{% block extrastyle %} +{{ block.super }} + + +{% endblock %} + +{% block extrahead %} +{{ block.super }} + + + +{% endblock %} + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› Orders +› Order +› {{ title }} +
    +{% endblock %} + + +{% block content %} +
    {% csrf_token %} +
    +
    + {% if bills %} + {% for account, total, lines in bills %} +
    +
    +
    +

    {{ account }}{{ total | floatformat:"-2" }} €

    + {% if not lines %} + + + + +
    {% trans 'Nothing to bill, all lines have size×quantity 0.' %}
    + {% else %} + + + + + + {% for line in lines %} + + + + + + + {% endfor %} + {% endif %} + +
    Description Period Size×Quantity Price
    + {{ line.order.description|safe }} + {% for discount in line.discounts %} +
          Discount per {{ discount.type }} + {% endfor %} +     		{{ line | periodformat }}{{ line.size | floatformat:"-2" }}×{{ line.metric | floatformat:"-2"}} +  {{ line.subtotal | floatformat:"-2" }} € + {% for discount in line.discounts %}
    {{ discount.total | floatformat:"-2" }} €{% endfor %} +
    +
    +
    +
    + {% endfor %} + {{ form.as_table }} + {% else %} + {{ form.as_admin }} + {% endif %} +
    + {% for obj in queryset %} + + {% endfor %} + + + +
    +
    +{% endblock %} diff --git a/orchestra/contrib/orders/templates/admin/orders/order/report.html b/orchestra/contrib/orders/templates/admin/orders/order/report.html new file mode 100644 index 00000000..87ef5f07 --- /dev/null +++ b/orchestra/contrib/orders/templates/admin/orders/order/report.html @@ -0,0 +1,62 @@ +{% load i18n utils %} + + + + Transaction Report + + + + + + + + + + + + + +{% for service, info in services %} + + + + + + + +{% endfor %} + + + + + + +
    {% trans "Services" %}{% trans "Active" %}{% trans "Cancelled" %}{% trans "Nominal price" %}{% trans "Number" %}
    {{ service }}{{ info.0 }}{{ info.1 }}{{ info.2 }}{{ info.3 }}
    {% trans "TOTAL" %}{{ totals.0 }}{{ totals.1 }}{{ totals.2 }}
    + + + diff --git a/orchestra/contrib/orders/templatetags/orders.py b/orchestra/contrib/orders/templatetags/orders.py new file mode 100644 index 00000000..8b7bf854 --- /dev/null +++ b/orchestra/contrib/orders/templatetags/orders.py @@ -0,0 +1,19 @@ +import datetime + +from django import template +from django.template.defaultfilters import date + + +register = template.Library() + + +@register.filter +def periodformat(line): + if line.ini == line.end: + return date(line.ini) + if line.ini.day == 1 and line.end.day == 1: + end = line.end - datetime.timedelta(days=1) + if line.ini.month == end.month: + return date(line.ini, "N Y") + return '%s to %s' % (date(line.ini, "N Y"), date(end, "N Y")) + return '%s to %s' % (date(line.ini), date(line.end)) diff --git a/orchestra/contrib/orders/tests/__init__.py b/orchestra/contrib/orders/tests/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/orchestra/contrib/orders/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/orchestra/contrib/payments/__init__.py b/orchestra/contrib/payments/__init__.py new file mode 100644 index 00000000..970bd432 --- /dev/null +++ b/orchestra/contrib/payments/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.payments.apps.PaymentsConfig' diff --git a/orchestra/contrib/payments/actions.py b/orchestra/contrib/payments/actions.py new file mode 100644 index 00000000..ae531555 --- /dev/null +++ b/orchestra/contrib/payments/actions.py @@ -0,0 +1,220 @@ +from functools import partial + +from django.contrib import messages +from django.contrib.admin import actions +from django.urls import reverse +from django.db import transaction +from django.shortcuts import render, redirect +from django.utils.safestring import mark_safe +from django.utils.text import capfirst +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.admin.decorators import action_with_confirmation +from orchestra.admin.utils import change_url + +from . import helpers +from .methods import PaymentMethod +from .models import Transaction + + +@transaction.atomic +def process_transactions(modeladmin, request, queryset): + processes = [] + if queryset.exclude(state=Transaction.WAITTING_PROCESSING).exists(): + messages.error(request, + _("Selected transactions must be on '{state}' state").format( + state=Transaction.WAITTING_PROCESSING) + ) + return + for method, transactions in queryset.group_by('source__method').items(): + if method is not None: + method = PaymentMethod.get(method) + procs = method.process(transactions) + processes += procs + for transaction in transactions: + modeladmin.log_change(request, transaction, _("Processed")) + if not processes: + return + opts = modeladmin.model._meta + num = len(queryset) + context = { + 'title': ngettext( + _("One selected transaction has been processed."), + _("%s Selected transactions have been processed.") % num, + num), + 'content_message': ngettext( + _("The following transaction process has been generated, " + "you may want to save it on your computer now."), + _("The following %s transaction processes have been generated, " + "you may want to save it on your computer now.") % len(processes), + len(processes)), + 'action_name': _("Process"), + 'processes': processes, + 'opts': opts, + 'app_label': opts.app_label, + } + return render(request, 'admin/payments/transaction/get_processes.html', context) + + +@transaction.atomic +@action_with_confirmation() +def mark_as_executed(modeladmin, request, queryset): + for transaction in queryset: + transaction.mark_as_executed() + modeladmin.log_change(request, transaction, _("Executed")) + num = len(queryset) + msg = ngettext( + _("One selected transaction has been marked as executed."), + _("%s selected transactions have been marked as executed.") % num, + num) + modeladmin.message_user(request, msg) +mark_as_executed.url_name = 'execute' +mark_as_executed.short_description = _("Mark as executed") + + +@transaction.atomic +@action_with_confirmation() +def mark_as_secured(modeladmin, request, queryset): + for transaction in queryset: + transaction.mark_as_secured() + modeladmin.log_change(request, transaction, _("Secured")) + num = len(queryset) + msg = ngettext( + _("One selected transaction has been marked as secured."), + _("%s selected transactions have been marked as secured.") % num, + num) + modeladmin.message_user(request, msg) +mark_as_secured.url_name = 'secure' +mark_as_secured.short_description = _("Mark as secured") + + +@transaction.atomic +@action_with_confirmation() +def mark_as_rejected(modeladmin, request, queryset): + for transaction in queryset: + transaction.mark_as_rejected() + modeladmin.log_change(request, transaction, _("Rejected")) + num = len(queryset) + msg = ngettext( + _("One selected transaction has been marked as rejected."), + _("%s selected transactions have been marked as rejected.") % num, + num) + modeladmin.message_user(request, msg) +mark_as_rejected.url_name = 'reject' +mark_as_rejected.short_description = _("Mark as rejected") + + +def _format_display_objects(modeladmin, request, queryset, related): + objects = [] + opts = modeladmin.model._meta + for obj in queryset: + objects.append( + mark_safe('{0}: {2}'.format( + capfirst(opts.verbose_name), change_url(obj), obj)) + ) + subobjects = [] + attr, verb = related + for trans in getattr(obj.transactions, attr)(): + subobjects.append( + mark_safe('Transaction: {} will be marked as {}'.format( + change_url(trans), trans, verb)) + ) + objects.append(subobjects) + return {'display_objects': objects} + +_format_executed = partial(_format_display_objects, related=('all', 'executed')) +_format_abort = partial(_format_display_objects, related=('processing', 'aborted')) +_format_commit = partial(_format_display_objects, related=('all', 'secured')) + + +@transaction.atomic +@action_with_confirmation(extra_context=_format_executed) +def mark_process_as_executed(modeladmin, request, queryset): + for process in queryset: + process.mark_as_executed() + modeladmin.log_change(request, process, _("Executed")) + num = len(queryset) + msg = ngettext( + _("One selected process has been marked as executed."), + _("%s selected processes have been marked as executed.") % num, + num) + modeladmin.message_user(request, msg) +mark_process_as_executed.url_name = 'executed' +mark_process_as_executed.short_description = _("Mark as executed") + + +@transaction.atomic +@action_with_confirmation(extra_context=_format_abort) +def abort(modeladmin, request, queryset): + for process in queryset: + process.abort() + modeladmin.log_change(request, process, _("Aborted")) + num = len(queryset) + msg = ngettext( + _("One selected process has been aborted."), + _("%s selected processes have been aborted.") % num, + num) + modeladmin.message_user(request, msg) +abort.url_name = 'abort' +abort.short_description = _("Abort") + + +@transaction.atomic +@action_with_confirmation(extra_context=_format_commit) +def commit(modeladmin, request, queryset): + for transaction in queryset: + transaction.mark_as_rejected() + modeladmin.log_change(request, transaction, _("Rejected")) + num = len(queryset) + msg = ngettext( + _("One selected transaction has been marked as rejected."), + _("%s selected transactions have been marked as rejected.") % num, + num) + modeladmin.message_user(request, msg) +commit.url_name = 'commit' +commit.short_description = _("Commit") + + +def report(modeladmin, request, queryset): + if queryset.model == Transaction: + transactions = queryset + else: + transactions = queryset.values_list('transactions__id', flat=True).distinct() + transactions = Transaction.objects.filter(id__in=transactions) + states = {} + total = 0 + transactions = transactions.order_by('bill__number') + for transaction in transactions: + state = transaction.get_state_display() + try: + states[state] += transaction.amount + except KeyError: + states[state] = transaction.amount + total += transaction.amount + context = { + 'states': states, + 'total': total, + 'transactions': transactions, + } + return render(request, 'admin/payments/transaction/report.html', context) + + +def reissue(modeladmin, request, queryset): + if len(queryset) != 1: + messages.error(request, _("One transaction should be selected.")) + return + trans = queryset[0] + if trans.state != trans.REJECTED: + messages.error(request, + _("Only rejected transactions can be reissued, " + "please reject current transaction if necessary.")) + return + url = reverse('admin:payments_transaction_add') + url += '?account=%i&bill=%i&source=%s&amount=%s¤cy=%s' % ( + trans.bill.account_id, + trans.bill_id, + trans.source_id or '', + trans.amount, + trans.currency, + ) + return redirect(url) diff --git a/orchestra/contrib/payments/admin.py b/orchestra/contrib/payments/admin.py new file mode 100644 index 00000000..e134c2e5 --- /dev/null +++ b/orchestra/contrib/payments/admin.py @@ -0,0 +1,245 @@ +from django.contrib import admin +from django.urls import reverse +from django.http import HttpResponseRedirect +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ChangeViewActionsMixin, ExtendedModelAdmin +from orchestra.admin.utils import admin_colored, admin_link, admin_date +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin +from orchestra.plugins.admin import SelectPluginAdminMixin + +from . import actions, helpers +from .methods import PaymentMethod +from .models import PaymentSource, Transaction, TransactionProcess + + +STATE_COLORS = { + Transaction.WAITTING_PROCESSING: 'darkorange', + Transaction.WAITTING_EXECUTION: 'magenta', + Transaction.EXECUTED: 'olive', + Transaction.SECURED: 'green', + Transaction.REJECTED: 'red', +} + +PROCESS_STATE_COLORS = { + TransactionProcess.CREATED: 'blue', + TransactionProcess.EXECUTED: 'olive', + TransactionProcess.ABORTED: 'red', + TransactionProcess.COMMITED: 'green', +} + + +class PaymentSourceAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): + list_display = ('label', 'method', 'number', 'account_link', 'is_active') + list_filter = ('method', 'is_active') + change_readonly_fields = ('method',) + search_fields = ('account__username', 'account__full_name', 'data') + plugin = PaymentMethod + plugin_field = 'method' + + +class TransactionInline(admin.TabularInline): + model = Transaction + can_delete = False + extra = 0 + fields = ( + 'transaction_link', 'bill_link', 'source_link', 'display_state', + 'amount', 'currency' + ) + readonly_fields = fields + + transaction_link = admin_link('__str__', short_description=_("ID")) + bill_link = admin_link('bill') + source_link = admin_link('source') + display_state = admin_colored('state', colors=STATE_COLORS) + + class Media: + css = { + 'all': ('orchestra/css/hide-inline-id.css',) + } + + def has_add_permission(self, *args, **kwargs): + return False + + def get_queryset(self, *args, **kwargs): + qs = super().get_queryset(*args, **kwargs) + return qs.select_related('source', 'bill') + + +class TransactionAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'id', 'bill_link', 'account_link', 'source_link', 'display_created_at', + 'display_modified_at', 'display_state', 'amount', 'process_link' + ) + list_filter = ('source__method', 'state') + fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ( + 'account_link', + 'bill_link', + 'source_link', + 'display_state', + 'amount', + 'currency', + 'process_link' + ) + }), + (_("Dates"), { + 'classes': ('wide',), + 'fields': ('display_created_at', 'display_modified_at'), + }), + ) + add_fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ( + 'bill', + 'source', + 'display_state', + 'amount', + 'currency', + ) + }), + ) + change_view_actions = ( + actions.process_transactions, actions.mark_as_executed, actions.mark_as_secured, + actions.mark_as_rejected, actions.reissue + ) + search_fields = ('bill__number', 'bill__account__username', 'id') + actions = change_view_actions + (actions.report, list_accounts,) + filter_by_account_fields = ('bill', 'source') + readonly_fields = ( + 'bill_link', 'display_state', 'process_link', 'account_link', 'source_link', + 'display_created_at', 'display_modified_at' + ) + list_select_related = ('source', 'bill__account', 'process') + date_hierarchy = 'created_at' + + bill_link = admin_link('bill') + source_link = admin_link('source') + process_link = admin_link('process', short_description=_("proc")) + account_link = admin_link('bill__account') + display_created_at = admin_date('created_at', short_description=_("Created")) + display_modified_at = admin_date('modified_at', short_description=_("Modified")) + + def has_delete_permission(self, *args, **kwargs): + return False + + def get_actions(self, request): + actions = super().get_actions(request) + if 'delete_selected' in actions: + del actions['delete_selected'] + return actions + + def get_change_readonly_fields(self, request, obj): + if obj.state in (Transaction.WAITTING_PROCESSING, Transaction.WAITTING_EXECUTION): + return () + return ('amount', 'currency') + + def get_change_view_actions(self, obj=None): + actions = super(TransactionAdmin, self).get_change_view_actions() + exclude = [] + if obj: + if obj.state == Transaction.WAITTING_PROCESSING: + exclude = ['mark_as_executed', 'mark_as_secured', 'reissue'] + elif obj.state == Transaction.WAITTING_EXECUTION: + exclude = ['process_transactions', 'mark_as_secured', 'reissue'] + if obj.state == Transaction.EXECUTED: + exclude = ['process_transactions', 'mark_as_executed', 'reissue'] + elif obj.state == Transaction.REJECTED: + exclude = ['process_transactions', 'mark_as_executed', 'mark_as_secured', 'mark_as_rejected'] + elif obj.state == Transaction.SECURED: + return [] + return [action for action in actions if action.__name__ not in exclude] + + @mark_safe + def display_state(self, obj): + state = admin_colored('state', colors=STATE_COLORS)(obj) + help_text = obj.get_state_help() + state = state.replace('{}', process.file.url, process.file.name) + file_url.admin_order_field = 'file' + + @mark_safe + def display_transactions(self, process): + ids = [] + lines = [] + counter = 0 + for trans in process.transactions.all(): + color = STATE_COLORS.get(trans.state, 'black') + state = trans.get_state_display() + ids.append('%i' % (color, state, trans.id)) + counter += 1 + len(str(trans.id)) + if counter > 100: + counter = 0 + lines.append(','.join(ids)) + ids = [] + lines.append(','.join(ids)) + transactions = '
    '.join(lines) + url = reverse('admin:payments_transaction_changelist') + url += '?process_id=%i' % process.id + return '%s' % (url, transactions) + display_transactions.short_description = _("Transactions") + + def has_add_permission(self, *args, **kwargs): + return False + + def get_change_view_actions(self, obj=None): + actions = super().get_change_view_actions() + exclude = [] + if obj: + if obj.state == TransactionProcess.EXECUTED: + exclude.append('mark_process_as_executed') + elif obj.state == TransactionProcess.COMMITED: + exclude = ['mark_process_as_executed', 'abort', 'commit'] + elif obj.state == TransactionProcess.ABORTED: + exclude = ['mark_process_as_executed', 'abort', 'commit'] + return [action for action in actions if action.__name__ not in exclude] + + def delete_view(self, request, object_id, extra_context=None): + queryset = self.model.objects.filter(id=object_id) + related_transactions = helpers.pre_delete_processes(self, request, queryset) + response = super().delete_view(request, object_id, extra_context) + if isinstance(response, HttpResponseRedirect): + helpers.post_delete_processes(self, request, related_transactions) + return response + + def delete_queryset(self, request, queryset): + # override default admin action delete behaviour + related_transactions = helpers.pre_delete_processes(self, request, queryset) + super().delete_queryset(self, request, queryset) + helpers.post_delete_processes(self, request, related_transactions) + + +admin.site.register(PaymentSource, PaymentSourceAdmin) +admin.site.register(Transaction, TransactionAdmin) +admin.site.register(TransactionProcess, TransactionProcessAdmin) diff --git a/orchestra/contrib/payments/api.py b/orchestra/contrib/payments/api.py new file mode 100644 index 00000000..7fd65f27 --- /dev/null +++ b/orchestra/contrib/payments/api.py @@ -0,0 +1,21 @@ +from rest_framework import viewsets + +from orchestra.api import router, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import PaymentSource, Transaction +from .serializers import PaymentSourceSerializer, TransactionSerializer + + +class PaymentSourceViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + serializer_class = PaymentSourceSerializer + queryset = PaymentSource.objects.all() + + +class TransactionViewSet(LogApiMixin, viewsets.ModelViewSet): + serializer_class = TransactionSerializer + queryset = Transaction.objects.all() + + +router.register(r'payment-sources', PaymentSourceViewSet) +router.register(r'transactions', TransactionViewSet) diff --git a/orchestra/contrib/payments/apps.py b/orchestra/contrib/payments/apps.py new file mode 100644 index 00000000..22eb60fa --- /dev/null +++ b/orchestra/contrib/payments/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + +from orchestra.core import accounts + + +class PaymentsConfig(AppConfig): + name = 'orchestra.contrib.payments' + verbose_name = "Payments" + + def ready(self): + from .models import PaymentSource, Transaction, TransactionProcess + accounts.register(PaymentSource, dashboard=False) + accounts.register(Transaction, icon='transaction.png', search=False) + accounts.register(TransactionProcess, icon='transactionprocess.png', dashboard=False, search=False) diff --git a/orchestra/contrib/payments/helpers.py b/orchestra/contrib/payments/helpers.py new file mode 100644 index 00000000..59163a09 --- /dev/null +++ b/orchestra/contrib/payments/helpers.py @@ -0,0 +1,36 @@ +from django.contrib import messages +from django.utils.translation import ngettext, gettext_lazy as _ + +from .models import Transaction + + +def pre_delete_processes(modeladmin, request, queryset): + if not queryset: + messages.warning(request, + _("No transaction process selected.")) + return + if queryset.exclude(transactions__state=Transaction.WAITTING_EXECUTION).exists(): + messages.error(request, + _("Done nothing. Not all related transactions in waitting execution.")) + return + # Store before deleting + related_transactions = [] + for process in queryset: + waitting_execution = process.transactions.filter(state=Transaction.WAITTING_EXECUTION) + related_transactions.extend(waitting_execution) + return related_transactions + + +def post_delete_processes(modeladmin, request, related_transactions): + # Confirmation + num = 0 + for transaction in related_transactions: + transaction.state = Transaction.WAITTING_PROCESSING + transaction.save(update_fields=('state', 'modified_at')) + num += 1 + modeladmin.log_change(request, transaction, _("Unprocessed")) + messages.success(request, ngettext( + "One related transaction has been marked as waitting for processing", + "%i related transactions have been marked as waitting for processing." % num, + num + )) diff --git a/orchestra/contrib/payments/locale/ca/LC_MESSAGES/django.mo b/orchestra/contrib/payments/locale/ca/LC_MESSAGES/django.mo new file mode 100644 index 00000000..647bb80b Binary files /dev/null and b/orchestra/contrib/payments/locale/ca/LC_MESSAGES/django.mo differ diff --git a/orchestra/contrib/payments/locale/ca/LC_MESSAGES/django.po b/orchestra/contrib/payments/locale/ca/LC_MESSAGES/django.po new file mode 100644 index 00000000..07f2e6a0 --- /dev/null +++ b/orchestra/contrib/payments/locale/ca/LC_MESSAGES/django.po @@ -0,0 +1,342 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2015-10-08 11:53+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: actions.py:24 +msgid "Selected transactions must be on '{state}' state" +msgstr "" + +#: actions.py:34 +msgid "Processed" +msgstr "" + +#: actions.py:41 +msgid "One selected transaction has been processed." +msgstr "" + +#: actions.py:42 +#, python-format +msgid "%s Selected transactions have been processed." +msgstr "" + +#: actions.py:45 +msgid "" +"The following transaction process has been generated, you may want to save " +"it on your computer now." +msgstr "" + +#: actions.py:47 +#, python-format +msgid "" +"The following %s transaction processes have been generated, you may want to " +"save it on your computer now." +msgstr "" + +#: actions.py:50 +msgid "Process" +msgstr "" + +#: actions.py:63 actions.py:134 models.py:97 models.py:172 +msgid "Executed" +msgstr "" + +#: actions.py:66 +msgid "One selected transaction has been marked as executed." +msgstr "" + +#: actions.py:67 +#, python-format +msgid "%s selected transactions have been marked as executed." +msgstr "" + +#: actions.py:71 actions.py:142 +msgid "Mark as executed" +msgstr "" + +#: actions.py:79 models.py:98 +msgid "Secured" +msgstr "" + +#: actions.py:82 +msgid "One selected transaction has been marked as secured." +msgstr "" + +#: actions.py:83 +#, python-format +msgid "%s selected transactions have been marked as secured." +msgstr "" + +#: actions.py:87 +msgid "Mark as secured" +msgstr "" + +#: actions.py:95 actions.py:166 models.py:99 +msgid "Rejected" +msgstr "" + +#: actions.py:98 actions.py:169 +msgid "One selected transaction has been marked as rejected." +msgstr "" + +#: actions.py:99 actions.py:170 +#, python-format +msgid "%s selected transactions have been marked as rejected." +msgstr "" + +#: actions.py:103 +msgid "Mark as rejected" +msgstr "" + +#: actions.py:137 +msgid "One selected process has been marked as executed." +msgstr "" + +#: actions.py:138 +#, python-format +msgid "%s selected processes have been marked as executed." +msgstr "" + +#: actions.py:150 models.py:173 +msgid "Aborted" +msgstr "" + +#: actions.py:153 +msgid "One selected process has been aborted." +msgstr "" + +#: actions.py:154 +#, python-format +msgid "%s selected processes have been aborted." +msgstr "" + +#: actions.py:158 +msgid "Abort" +msgstr "" + +#: actions.py:174 +msgid "Commit" +msgstr "" + +#: admin.py:44 +msgid "ID" +msgstr "" + +#: admin.py:106 +msgid "proc" +msgstr "" + +#: admin.py:129 templates/admin/payments/transaction/report.html:62 +msgid "State" +msgstr "" + +#: admin.py:168 +msgid "Transactions" +msgstr "" + +#: helpers.py:11 +msgid "No transaction process selected." +msgstr "" + +#: helpers.py:15 +msgid "Done nothing. Not all related transactions in waitting execution." +msgstr "" + +#: helpers.py:32 +msgid "Unprocessed" +msgstr "" + +#: helpers.py:34 +#, python-format +msgid "" +"One related transaction has been marked as waitting for processing" +msgid_plural "" +"%i related transactions have been marked as waitting for processing." +msgstr[0] "" +msgstr[1] "" + +#: methods/creditcard.py:11 +msgid "Label" +msgstr "" + +#: methods/creditcard.py:12 +msgid "Use a name such as \"Jo's Visa\" to remember which card it is." +msgstr "" + +#: methods/creditcard.py:30 +msgid "Credit card" +msgstr "" + +#: methods/sepadirectdebit.py:23 methods/sepadirectdebit.py:30 +msgid "Name" +msgstr "" + +#: methods/sepadirectdebit.py:39 +msgid "SEPA Direct Debit" +msgstr "" + +#: methods/sepadirectdebit.py:47 +msgid "" +"The transaction is created and requires the generation of the SEPA direct " +"debit XML file." +msgstr "" + +#: methods/sepadirectdebit.py:49 +msgid "" +"SEPA Direct Debit XML file is generated but needs to be sent to the " +"financial institution." +msgstr "" + +#: models.py:20 +msgid "account" +msgstr "" + +#: models.py:22 +msgid "method" +msgstr "" + +#: models.py:24 models.py:177 +msgid "data" +msgstr "" + +#: models.py:25 +msgid "active" +msgstr "" + +#: models.py:95 +msgid "Waitting processing" +msgstr "" + +#: models.py:96 +msgid "Waitting execution" +msgstr "" + +#: models.py:102 +msgid "" +"The transaction is created and requires processing by the specific payment " +"method." +msgstr "" + +#: models.py:104 +msgid "" +"The transaction is processed and its pending execution on the related " +"financial institution." +msgstr "" + +#: models.py:106 +msgid "The transaction is executed on the financial institution." +msgstr "" + +#: models.py:107 +msgid "The transaction ammount is secured." +msgstr "" + +#: models.py:108 +msgid "" +"The transaction has failed and the ammount is lost, a new transaction should " +"be created for recharging." +msgstr "" + +#: models.py:112 +msgid "bill" +msgstr "" + +#: models.py:115 +msgid "source" +msgstr "" + +#: models.py:117 +msgid "process" +msgstr "" + +#: models.py:118 models.py:179 +msgid "state" +msgstr "" + +#: models.py:120 +msgid "amount" +msgstr "" + +#: models.py:122 models.py:180 +msgid "created" +msgstr "" + +#: models.py:123 +msgid "modified" +msgstr "" + +#: models.py:138 +msgid "New transactions can not be allocated for this bill." +msgstr "" + +#: models.py:171 templates/admin/payments/transaction/report.html:63 +msgid "Created" +msgstr "" + +#: models.py:174 +msgid "Commited" +msgstr "" + +#: models.py:178 +msgid "file" +msgstr "" + +#: models.py:181 +msgid "updated" +msgstr "" + +#: models.py:184 +msgid "Transaction processes" +msgstr "" + +#: settings.py:14 +msgid "" +"Direct debit, this bill will be automatically charged to " +"your bank account with IBAN number
    %(number)s." +msgstr "" +"Càrrec per domiciliació, aquesta factura es cobrarà " +"automaticament en el teu compte bancari amb IBAN
    %(number)s." + +#: templates/admin/payments/transaction/report.html:38 +msgid "Summary" +msgstr "" + +#: templates/admin/payments/transaction/report.html:39 +#: templates/admin/payments/transaction/report.html:61 +msgid "Amount" +msgstr "" + +#: templates/admin/payments/transaction/report.html:48 +msgid "TOTAL" +msgstr "" + +#: templates/admin/payments/transaction/report.html:57 +msgid "Bill" +msgstr "" + +#: templates/admin/payments/transaction/report.html:58 +msgid "Account" +msgstr "" + +#: templates/admin/payments/transaction/report.html:59 +msgid "Contact" +msgstr "" + +#: templates/admin/payments/transaction/report.html:64 +msgid "Updated" +msgstr "" diff --git a/orchestra/contrib/payments/locale/es/LC_MESSAGES/django.mo b/orchestra/contrib/payments/locale/es/LC_MESSAGES/django.mo new file mode 100644 index 00000000..de1e91c1 Binary files /dev/null and b/orchestra/contrib/payments/locale/es/LC_MESSAGES/django.mo differ diff --git a/orchestra/contrib/payments/locale/es/LC_MESSAGES/django.po b/orchestra/contrib/payments/locale/es/LC_MESSAGES/django.po new file mode 100644 index 00000000..58e0828c --- /dev/null +++ b/orchestra/contrib/payments/locale/es/LC_MESSAGES/django.po @@ -0,0 +1,342 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2015-10-08 12:14+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: actions.py:24 +msgid "Selected transactions must be on '{state}' state" +msgstr "" + +#: actions.py:34 +msgid "Processed" +msgstr "" + +#: actions.py:41 +msgid "One selected transaction has been processed." +msgstr "" + +#: actions.py:42 +#, python-format +msgid "%s Selected transactions have been processed." +msgstr "" + +#: actions.py:45 +msgid "" +"The following transaction process has been generated, you may want to save " +"it on your computer now." +msgstr "" + +#: actions.py:47 +#, python-format +msgid "" +"The following %s transaction processes have been generated, you may want to " +"save it on your computer now." +msgstr "" + +#: actions.py:50 +msgid "Process" +msgstr "" + +#: actions.py:63 actions.py:134 models.py:97 models.py:172 +msgid "Executed" +msgstr "" + +#: actions.py:66 +msgid "One selected transaction has been marked as executed." +msgstr "" + +#: actions.py:67 +#, python-format +msgid "%s selected transactions have been marked as executed." +msgstr "" + +#: actions.py:71 actions.py:142 +msgid "Mark as executed" +msgstr "" + +#: actions.py:79 models.py:98 +msgid "Secured" +msgstr "" + +#: actions.py:82 +msgid "One selected transaction has been marked as secured." +msgstr "" + +#: actions.py:83 +#, python-format +msgid "%s selected transactions have been marked as secured." +msgstr "" + +#: actions.py:87 +msgid "Mark as secured" +msgstr "" + +#: actions.py:95 actions.py:166 models.py:99 +msgid "Rejected" +msgstr "" + +#: actions.py:98 actions.py:169 +msgid "One selected transaction has been marked as rejected." +msgstr "" + +#: actions.py:99 actions.py:170 +#, python-format +msgid "%s selected transactions have been marked as rejected." +msgstr "" + +#: actions.py:103 +msgid "Mark as rejected" +msgstr "" + +#: actions.py:137 +msgid "One selected process has been marked as executed." +msgstr "" + +#: actions.py:138 +#, python-format +msgid "%s selected processes have been marked as executed." +msgstr "" + +#: actions.py:150 models.py:173 +msgid "Aborted" +msgstr "" + +#: actions.py:153 +msgid "One selected process has been aborted." +msgstr "" + +#: actions.py:154 +#, python-format +msgid "%s selected processes have been aborted." +msgstr "" + +#: actions.py:158 +msgid "Abort" +msgstr "" + +#: actions.py:174 +msgid "Commit" +msgstr "" + +#: admin.py:44 +msgid "ID" +msgstr "" + +#: admin.py:106 +msgid "proc" +msgstr "" + +#: admin.py:129 templates/admin/payments/transaction/report.html:62 +msgid "State" +msgstr "" + +#: admin.py:168 +msgid "Transactions" +msgstr "" + +#: helpers.py:11 +msgid "No transaction process selected." +msgstr "" + +#: helpers.py:15 +msgid "Done nothing. Not all related transactions in waitting execution." +msgstr "" + +#: helpers.py:32 +msgid "Unprocessed" +msgstr "" + +#: helpers.py:34 +#, python-format +msgid "" +"One related transaction has been marked as waitting for processing" +msgid_plural "" +"%i related transactions have been marked as waitting for processing." +msgstr[0] "" +msgstr[1] "" + +#: methods/creditcard.py:11 +msgid "Label" +msgstr "" + +#: methods/creditcard.py:12 +msgid "Use a name such as \"Jo's Visa\" to remember which card it is." +msgstr "" + +#: methods/creditcard.py:30 +msgid "Credit card" +msgstr "" + +#: methods/sepadirectdebit.py:23 methods/sepadirectdebit.py:30 +msgid "Name" +msgstr "" + +#: methods/sepadirectdebit.py:39 +msgid "SEPA Direct Debit" +msgstr "" + +#: methods/sepadirectdebit.py:47 +msgid "" +"The transaction is created and requires the generation of the SEPA direct " +"debit XML file." +msgstr "" + +#: methods/sepadirectdebit.py:49 +msgid "" +"SEPA Direct Debit XML file is generated but needs to be sent to the " +"financial institution." +msgstr "" + +#: models.py:20 +msgid "account" +msgstr "" + +#: models.py:22 +msgid "method" +msgstr "" + +#: models.py:24 models.py:177 +msgid "data" +msgstr "" + +#: models.py:25 +msgid "active" +msgstr "" + +#: models.py:95 +msgid "Waitting processing" +msgstr "" + +#: models.py:96 +msgid "Waitting execution" +msgstr "" + +#: models.py:102 +msgid "" +"The transaction is created and requires processing by the specific payment " +"method." +msgstr "" + +#: models.py:104 +msgid "" +"The transaction is processed and its pending execution on the related " +"financial institution." +msgstr "" + +#: models.py:106 +msgid "The transaction is executed on the financial institution." +msgstr "" + +#: models.py:107 +msgid "The transaction ammount is secured." +msgstr "" + +#: models.py:108 +msgid "" +"The transaction has failed and the ammount is lost, a new transaction should " +"be created for recharging." +msgstr "" + +#: models.py:112 +msgid "bill" +msgstr "" + +#: models.py:115 +msgid "source" +msgstr "" + +#: models.py:117 +msgid "process" +msgstr "" + +#: models.py:118 models.py:179 +msgid "state" +msgstr "" + +#: models.py:120 +msgid "amount" +msgstr "" + +#: models.py:122 models.py:180 +msgid "created" +msgstr "" + +#: models.py:123 +msgid "modified" +msgstr "" + +#: models.py:138 +msgid "New transactions can not be allocated for this bill." +msgstr "" + +#: models.py:171 templates/admin/payments/transaction/report.html:63 +msgid "Created" +msgstr "" + +#: models.py:174 +msgid "Commited" +msgstr "" + +#: models.py:178 +msgid "file" +msgstr "" + +#: models.py:181 +msgid "updated" +msgstr "" + +#: models.py:184 +msgid "Transaction processes" +msgstr "" + +#: settings.py:14 +msgid "" +"Direct debit, this bill will be automatically charged to " +"your bank account with IBAN number
    %(number)s." +msgstr "" +"Adeudo por domiciliación, esta factura se cobrará " +"automaticamente en tu cuenta bancaria con IBAN
    %(number)s." + +#: templates/admin/payments/transaction/report.html:38 +msgid "Summary" +msgstr "" + +#: templates/admin/payments/transaction/report.html:39 +#: templates/admin/payments/transaction/report.html:61 +msgid "Amount" +msgstr "" + +#: templates/admin/payments/transaction/report.html:48 +msgid "TOTAL" +msgstr "" + +#: templates/admin/payments/transaction/report.html:57 +msgid "Bill" +msgstr "" + +#: templates/admin/payments/transaction/report.html:58 +msgid "Account" +msgstr "" + +#: templates/admin/payments/transaction/report.html:59 +msgid "Contact" +msgstr "" + +#: templates/admin/payments/transaction/report.html:64 +msgid "Updated" +msgstr "" diff --git a/orchestra/contrib/payments/methods/__init__.py b/orchestra/contrib/payments/methods/__init__.py new file mode 100644 index 00000000..9a9aaa55 --- /dev/null +++ b/orchestra/contrib/payments/methods/__init__.py @@ -0,0 +1 @@ +from .options import PaymentMethod diff --git a/orchestra/contrib/payments/methods/creditcard.py b/orchestra/contrib/payments/methods/creditcard.py new file mode 100644 index 00000000..14e5851c --- /dev/null +++ b/orchestra/contrib/payments/methods/creditcard.py @@ -0,0 +1,32 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.plugins.forms import PluginDataForm + +from .options import PaymentMethod + + +class CreditCardForm(PluginDataForm): + label = forms.CharField(max_length=128, label=_("Label"), + help_text=_("Use a name such as \"Jo's Visa\" to remember which " + "card it is.")) + first_name = forms.CharField(max_length=128) + last_name = forms.CharField(max_length=128) + address = forms.CharField(max_length=128) + zip = forms.CharField(max_length=128) + city = forms.CharField(max_length=128) + country = forms.CharField(max_length=128) + card_number = forms.CharField(max_length=128) + expiration_date = forms.CharField(max_length=128) + security_code = forms.CharField(max_length=128) + + +class CreditCardSerializer(serializers.Serializer): + pass + + +class CreditCard(PaymentMethod): + verbose_name = _("Credit card") + form = CreditCardForm + serializer = CreditCardSerializer diff --git a/orchestra/contrib/payments/methods/options.py b/orchestra/contrib/payments/methods/options.py new file mode 100644 index 00000000..5e19cbd6 --- /dev/null +++ b/orchestra/contrib/payments/methods/options.py @@ -0,0 +1,42 @@ +import importlib +import logging +import os +from dateutil import relativedelta +from functools import lru_cache + +from orchestra import plugins +from orchestra.utils.python import import_class + +from .. import settings + + +class PaymentMethod(plugins.Plugin, metaclass=plugins.PluginMount): + label_field = 'label' + number_field = 'number' + allow_recharge = False + due_delta = relativedelta.relativedelta(months=1) + plugin_field = 'method' + state_help = {} + + @classmethod + @lru_cache() + def get_plugins(cls, all=False): + if all: + for module in os.listdir(os.path.dirname(__file__)): + if module not in ('options.py', '__init__.py') and module[-3:] == '.py': + importlib.import_module('.'+module[:-3], __package__) + plugins = super().get_plugins() + else: + plugins = [] + for cls in settings.PAYMENTS_ENABLED_METHODS: + plugins.append(import_class(cls)) + return plugins + + def get_label(self): + return self.instance.data[self.label_field] + + def get_number(self): + return self.instance.data[self.number_field] + + def get_bill_message(self): + return '' diff --git a/orchestra/contrib/payments/methods/pain.001.001.03.xsd b/orchestra/contrib/payments/methods/pain.001.001.03.xsd new file mode 100644 index 00000000..4f65ddcc --- /dev/null +++ b/orchestra/contrib/payments/methods/pain.001.001.03.xsd @@ -0,0 +1,921 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/orchestra/contrib/payments/methods/pain.008.001.02.xsd b/orchestra/contrib/payments/methods/pain.008.001.02.xsd new file mode 100644 index 00000000..394b8045 --- /dev/null +++ b/orchestra/contrib/payments/methods/pain.008.001.02.xsd @@ -0,0 +1,879 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/orchestra/contrib/payments/methods/sepadirectdebit.py b/orchestra/contrib/payments/methods/sepadirectdebit.py new file mode 100644 index 00000000..8ce1bf31 --- /dev/null +++ b/orchestra/contrib/payments/methods/sepadirectdebit.py @@ -0,0 +1,319 @@ +import datetime +import logging +import os +from io import StringIO + +from django import forms +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from django_iban.validators import IBANValidator, IBAN_COUNTRY_CODE_LENGTH +from rest_framework import serializers + +from orchestra.plugins.forms import PluginDataForm + +from .. import settings +from .options import PaymentMethod + + +logger = logging.getLogger(__name__) + +try: + import lxml +except ImportError: + logger.error('Error loading lxml, module not installed.') + + +class SEPADirectDebitForm(PluginDataForm): + iban = forms.CharField(label='IBAN', + widget=forms.TextInput(attrs={'size': '50'})) + name = forms.CharField(max_length=128, label=_("Name"), + widget=forms.TextInput(attrs={'size': '50'})) + + +class SEPADirectDebitSerializer(serializers.Serializer): + iban = serializers.CharField(label='IBAN', validators=[IBANValidator()], + min_length=min(IBAN_COUNTRY_CODE_LENGTH.values()), max_length=34) + name = serializers.CharField(label=_("Name"), max_length=128) + + def validate(self, data): + data['iban'] = data['iban'].strip() + data['name'] = data['name'].strip() + return data + + +class SEPADirectDebit(PaymentMethod): + verbose_name = _("SEPA Direct Debit") + label_field = 'name' + number_field = 'iban' + allow_recharge = True + form = SEPADirectDebitForm + serializer = SEPADirectDebitSerializer + due_delta = datetime.timedelta(days=5) + state_help = { + 'WAITTING_PROCESSING': _("The transaction is created and requires the generation of " + "the SEPA direct debit XML file."), + 'WAITTING_EXECUTION': _("SEPA Direct Debit XML file is generated but needs to be sent " + "to the financial institution."), + } + + def get_bill_message(self): + context = { + 'number': self.instance.number + } + return settings.PAYMENTS_DD_BILL_MESSAGE % context + + @classmethod + def process(cls, transactions): + debts = [] + credits = [] + for transaction in transactions: + if transaction.amount < 0: + credits.append(transaction) + else: + debts.append(transaction) + processes = [] + if debts: + proc = cls.process_debts(debts) + processes.append(proc) + if credits: + proc = cls.process_credits(credits) + processes.append(proc) + return processes + + @classmethod + def process_credits(cls, transactions): + import lxml.builder + from lxml.builder import E + from ..models import TransactionProcess + process = TransactionProcess.objects.create() + context = cls.get_context(transactions) + # http://businessbanking.bankofireland.com/fs/doc/wysiwyg/b22440-mss130725-pain001-xml-file-structure-dec13.pdf + sepa = lxml.builder.ElementMaker( + nsmap = { + 'xsi': 'http://www.w3.org/2001/XMLSchema-instance', + None: 'urn:iso:std:iso:20022:tech:xsd:pain.001.001.03', + } + ) + sepa = sepa.Document( + E.CstmrCdtTrfInitn( + cls.get_header(context, process), + E.PmtInf( # Payment Info + E.PmtInfId(str(process.id)), # Payment Id + E.PmtMtd("TRF"), # Payment Method + E.NbOfTxs(context['num_transactions']), # Number of Transactions + E.CtrlSum(context['total']), # Control Sum + E.ReqdExctnDt( # Requested Execution Date + (context['now']+datetime.timedelta(days=10)).strftime("%Y-%m-%d") + ), + E.Dbtr( # Debtor + E.Nm(context['name']) + ), + E.DbtrAcct( # Debtor Account + E.Id( + E.IBAN(context['iban']) + ) + ), + E.DbtrAgt( # Debtor Agent + E.FinInstnId( # Financial Institution Id + E.BIC(context['bic']) + ) + ), + *list(cls.get_credit_transactions(transactions, process)) # Transactions + ) + ) + ) + file_name = 'credit-transfer-%i.xml' % process.id + cls.process_xml(sepa, 'pain.001.001.03.xsd', file_name, process) + return process + + @classmethod + def process_debts(cls, transactions): + import lxml.builder + from lxml.builder import E + from ..models import TransactionProcess + process = TransactionProcess.objects.create() + context = cls.get_context(transactions) + # http://businessbanking.bankofireland.com/fs/doc/wysiwyg/sepa-direct-debit-pain-008-001-02-xml-file-structure-july-2013.pdf + sepa = lxml.builder.ElementMaker( + nsmap = { + 'xsi': 'http://www.w3.org/2001/XMLSchema-instance', + None: 'urn:iso:std:iso:20022:tech:xsd:pain.008.001.02', + } + ) + sepa = sepa.Document( + E.CstmrDrctDbtInitn( + cls.get_header(context, process), + E.PmtInf( # Payment Info + E.PmtInfId(str(process.id)), # Payment Id + E.PmtMtd("DD"), # Payment Method + E.NbOfTxs(context['num_transactions']), # Number of Transactions + E.CtrlSum(context['total']), # Control Sum + E.PmtTpInf( # Payment Type Info + E.SvcLvl( # Service Level + E.Cd("SEPA") # Code + ), + E.LclInstrm( # Local Instrument + E.Cd("CORE") # Code + ), + E.SeqTp("RCUR") # Sequence Type + ), + E.ReqdColltnDt( # Requested Collection Date + context['now'].strftime("%Y-%m-%d") + ), + E.Cdtr( # Creditor + E.Nm(context['name']) + ), + E.CdtrAcct( # Creditor Account + E.Id( + E.IBAN(context['iban']) + ) + ), + E.CdtrAgt( # Creditor Agent + E.FinInstnId( # Financial Institution Id + E.BIC(context['bic']) + ) + ), + *list(cls.get_debt_transactions(transactions, process)) # Transactions + ) + ) + ) + file_name = 'direct-debit-%i.xml' % process.id + cls.process_xml(sepa, 'pain.008.001.02.xsd', file_name, process) + return process + + @classmethod + def get_context(cls, transactions): + return { + 'name': settings.PAYMENTS_DD_CREDITOR_NAME, + 'iban': settings.PAYMENTS_DD_CREDITOR_IBAN, + 'bic': settings.PAYMENTS_DD_CREDITOR_BIC, + 'at02_id': settings.PAYMENTS_DD_CREDITOR_AT02_ID, + 'now': timezone.now(), + 'total': str(sum([abs(transaction.amount) for transaction in transactions])), + 'num_transactions': str(len(transactions)), + } + + @classmethod + def get_debt_transactions(cls, transactions, process): + import lxml.builder + from lxml.builder import E + for transaction in transactions: + transaction.process = process + transaction.state = transaction.WAITTING_EXECUTION + transaction.save(update_fields=('state', 'process', 'modified_at')) + account = transaction.account + data = transaction.source.data + yield E.DrctDbtTxInf( # Direct Debit Transaction Info + E.PmtId( # Payment Id + E.EndToEndId( # Payment Id/End to End + str(transaction.bill.number)+'-'+str(transaction.id) + ) + ), + E.InstdAmt( # Instructed Amount + str(abs(transaction.amount)), + Ccy=transaction.currency.upper() + ), + E.DrctDbtTx( # Direct Debit Transaction + E.MndtRltdInf( # Mandate Related Info + # + 10000 xk vam canviar de sistema per generar aquestes IDs i volem evitar colisions amb els + # numeros usats antigament + E.MndtId(str(transaction.source_id+10000)), # Mandate Id + E.DtOfSgntr( # Date of Signature + account.date_joined.strftime("%Y-%m-%d") + ) + ) + ), + E.DbtrAgt( # Debtor Agent + E.FinInstnId( # Financial Institution Id + E.Othr( + E.Id('NOTPROVIDED') + ) + ) + ), + E.Dbtr( # Debtor + E.Nm(account.billcontact.get_name()), # Name + ), + E.DbtrAcct( # Debtor Account + E.Id( + E.IBAN(data['iban'].replace(' ', '')) + ), + ), + ) + + @classmethod + def get_credit_transactions(cls, transactions, process): + import lxml.builder + from lxml.builder import E + for transaction in transactions: + transaction.process = process + transaction.state = transaction.WAITTING_EXECUTION + transaction.save(update_fields=('state', 'process', 'modified_at')) + account = transaction.account + data = transaction.source.data + yield E.CdtTrfTxInf( # Credit Transfer Transaction Info + E.PmtId( # Payment Id + E.EndToEndId(str(transaction.id)) # Payment Id/End to End + ), + E.Amt( # Amount + E.InstdAmt( # Instructed Amount + str(abs(transaction.amount)), + Ccy=transaction.currency.upper() + ) + ), + E.CdtrAgt( # Creditor Agent + E.FinInstnId( # Financial Institution Id + E.Othr( + E.Id('NOTPROVIDED') + ) + ) + ), + E.Cdtr( # Debtor + E.Nm(account.name), # Name + ), + E.CdtrAcct( # Creditor Account + E.Id( + E.IBAN(data['iban'].replace(' ', '')) + ), + ), + ) + + @classmethod + def get_header(cls, context, process): + import lxml.builder + from lxml.builder import E + return E.GrpHdr( # Group Header + E.MsgId(str(process.id)), # Message Id + E.CreDtTm( # Creation Date Time + context['now'].strftime("%Y-%m-%dT%H:%M:%S") + ), + E.NbOfTxs(context['num_transactions']), # Number of Transactions + E.CtrlSum(context['total']), # Control Sum + E.InitgPty( # Initiating Party + E.Nm(context['name']), # Name + E.Id( # Identification + E.OrgId( # Organisation Id + E.Othr( + E.Id(context['at02_id']) + ) + ) + ) + ) + ) + + @classmethod + def process_xml(cls, sepa, xsd, file_name, process): + from lxml import etree + # http://www.iso20022.org/documents/messages/1_0_version/pain/schemas/pain.008.001.02.zip + path = os.path.dirname(os.path.realpath(__file__)) + xsd_path = os.path.join(path, xsd) + schema_doc = etree.parse(xsd_path) + schema = etree.XMLSchema(schema_doc) + sepa = StringIO(etree.tostring(sepa).decode('utf8')) + sepa = etree.parse(sepa) + schema.assertValid(sepa) + process.file = file_name + process.save(update_fields=['file']) + sepa.write(process.file.path, + pretty_print=True, + xml_declaration=True, + encoding='UTF-8') diff --git a/orchestra/contrib/payments/models.py b/orchestra/contrib/payments/models.py new file mode 100644 index 00000000..9617ce40 --- /dev/null +++ b/orchestra/contrib/payments/models.py @@ -0,0 +1,210 @@ +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ +from jsonfield import JSONField + +from orchestra.models.fields import PrivateFileField +from orchestra.models.queryset import group_by + +from . import settings +from .methods import PaymentMethod + + +class PaymentSourcesQueryset(models.QuerySet): + def get_default(self): + return self.filter(is_active=True).first() + + +class PaymentSource(models.Model): + account = models.ForeignKey('accounts.Account', verbose_name=_("account"), + related_name='paymentsources', on_delete=models.CASCADE) + method = models.CharField(_("method"), max_length=32, + choices=PaymentMethod.get_choices()) + data = JSONField(_("data"), default={}) + is_active = models.BooleanField(_("active"), default=True) + + objects = PaymentSourcesQueryset.as_manager() + + def __str__(self): + return "%s (%s)" % (self.label, self.method_class.verbose_name) + + @cached_property + def method_class(self): + return PaymentMethod.get(self.method) + + @cached_property + def method_instance(self): + """ Per request lived method_instance """ + return self.method_class(self) + + @cached_property + def label(self): + return self.method_instance.get_label() + + @cached_property + def number(self): + return self.method_instance.get_number() + + def get_bill_context(self): + method = self.method_instance + return { + 'message': method.get_bill_message(), + } + + def get_due_delta(self): + return self.method_instance.due_delta + + def clean(self): + self.data = self.method_instance.clean_data() + + +class TransactionQuerySet(models.QuerySet): + group_by = group_by + + def create(self, **kwargs): + source = kwargs.get('source') + if source is None or not hasattr(source.method_class, 'process'): + # Manual payments don't need processing + kwargs['state'] = self.model.WAITTING_EXECUTION + amount = kwargs.get('amount') + if amount == 0: + kwargs['state'] = self.model.SECURED + return super(TransactionQuerySet, self).create(**kwargs) + + def secured(self): + return self.filter(state=Transaction.SECURED) + + def exclude_rejected(self): + return self.exclude(state=Transaction.REJECTED) + + def amount(self): + return next(iter(self.aggregate(models.Sum('amount')).values())) or 0 + + def processing(self): + return self.filter(state__in=[Transaction.EXECUTED, Transaction.WAITTING_EXECUTION]) + + +class Transaction(models.Model): + WAITTING_PROCESSING = 'WAITTING_PROCESSING' # CREATED + WAITTING_EXECUTION = 'WAITTING_EXECUTION' # PROCESSED + EXECUTED = 'EXECUTED' + SECURED = 'SECURED' + REJECTED = 'REJECTED' + STATES = ( + (WAITTING_PROCESSING, _("Waitting processing")), + (WAITTING_EXECUTION, _("Waitting execution")), + (EXECUTED, _("Executed")), + (SECURED, _("Secured")), + (REJECTED, _("Rejected")), + ) + STATE_HELP = { + WAITTING_PROCESSING: _("The transaction is created and requires processing by the " + "specific payment method."), + WAITTING_EXECUTION: _("The transaction is processed and its pending execution on " + "the related financial institution."), + EXECUTED: _("The transaction is executed on the financial institution."), + SECURED: _("The transaction ammount is secured."), + REJECTED: _("The transaction has failed and the ammount is lost, a new transaction " + "should be created for recharging."), + } + + bill = models.ForeignKey('bills.bill', on_delete=models.CASCADE, verbose_name=_("bill"), + related_name='transactions') + source = models.ForeignKey(PaymentSource, null=True, blank=True, on_delete=models.SET_NULL, + verbose_name=_("source"), related_name='transactions') + process = models.ForeignKey('payments.TransactionProcess', null=True, blank=True, + on_delete=models.SET_NULL, verbose_name=_("process"), related_name='transactions') + state = models.CharField(_("state"), max_length=32, choices=STATES, + default=WAITTING_PROCESSING) + amount = models.DecimalField(_("amount"), max_digits=12, decimal_places=2) + currency = models.CharField(max_length=10, default=settings.PAYMENT_CURRENCY) + created_at = models.DateTimeField(_("created"), auto_now_add=True) + modified_at = models.DateTimeField(_("modified"), auto_now=True) + + objects = TransactionQuerySet.as_manager() + + def __str__(self): + return "#%i" % self.id + + @property + def account(self): + return self.bill.account + + def clean(self): + if not self.pk: + amount = self.bill.transactions.exclude(state=self.REJECTED).amount() + if amount >= self.bill.total: + raise ValidationError( + _("Bill %(number)s already has valid transactions that cover bill total amount (%(amount)s).") % { + 'number': self.bill.number, + 'amount': amount, + } + ) + + def get_state_help(self): + if self.source: + return self.source.method_instance.state_help.get(self.state) or self.STATE_HELP.get(self.state) + return self.STATE_HELP.get(self.state) + + def mark_as_processed(self): + self.state = self.WAITTING_EXECUTION + self.save(update_fields=('state', 'modified_at')) + + def mark_as_executed(self): + self.state = self.EXECUTED + self.save(update_fields=('state', 'modified_at')) + + def mark_as_secured(self): + self.state = self.SECURED + self.save(update_fields=('state', 'modified_at')) + + def mark_as_rejected(self): + self.state = self.REJECTED + self.save(update_fields=('state', 'modified_at')) + + +class TransactionProcess(models.Model): + """ + Stores arbitrary data generated by payment methods while processing transactions + """ + CREATED = 'CREATED' + EXECUTED = 'EXECUTED' + ABORTED = 'ABORTED' + COMMITED = 'COMMITED' + STATES = ( + (CREATED, _("Created")), + (EXECUTED, _("Executed")), + (ABORTED, _("Aborted")), + (COMMITED, _("Commited")), + ) + + data = JSONField(_("data"), blank=True) + file = PrivateFileField(_("file"), blank=True) + state = models.CharField(_("state"), max_length=16, choices=STATES, default=CREATED) + created_at = models.DateTimeField(_("created"), auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(_("updated"), auto_now=True) + + class Meta: + verbose_name_plural = _("Transaction processes") + + def __str__(self): + return '#%i' % self.id + + def mark_as_executed(self): + self.state = self.EXECUTED + for transaction in self.transactions.all(): + transaction.mark_as_executed() + self.save(update_fields=('state', 'updated_at')) + + def abort(self): + self.state = self.ABORTED + for transaction in self.transactions.all(): + transaction.mark_as_rejected() + self.save(update_fields=('state', 'updated_at')) + + def commit(self): + self.state = self.COMMITED + for transaction in self.transactions.processing(): + transaction.mark_as_secured() + self.save(update_fields=('state', 'updated_at')) diff --git a/orchestra/contrib/payments/serializers.py b/orchestra/contrib/payments/serializers.py new file mode 100644 index 00000000..93ae9f78 --- /dev/null +++ b/orchestra/contrib/payments/serializers.py @@ -0,0 +1,46 @@ +from rest_framework import serializers + +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .methods import PaymentMethod +from .models import PaymentSource, Transaction + + +class PaymentSourceSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + class Meta: + model = PaymentSource + fields = ('url', 'id', 'method', 'data', 'is_active') + + def validate(self, data): + """ validate data according to method """ + data = super(PaymentSourceSerializer, self).validate(data) + plugin = PaymentMethod.get(data['method']) + serializer_class = plugin().get_serializer() + serializer = serializer_class(data=data['data']) + if not serializer.is_valid(): + raise serializers.ValidationError(serializer.errors) + return data + + def transform_data(self, obj, value): + if not obj: + return {} + if obj.method: + plugin = PaymentMethod.get(obj.method) + serializer_class = plugin().get_serializer() + return serializer_class().to_native(obj.data) + return obj.data + + # TODO + def metadata(self): + meta = super(PaymentSourceSerializer, self).metadata() + meta['data'] = { + method.get_name(): method().get_serializer()().metadata() + for method in PaymentMethod.get_plugins() + } + return meta + + +class TransactionSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): + class Meta: + model = Transaction + exclude = ('process',) diff --git a/orchestra/contrib/payments/settings.py b/orchestra/contrib/payments/settings.py new file mode 100644 index 00000000..65e5702d --- /dev/null +++ b/orchestra/contrib/payments/settings.py @@ -0,0 +1,46 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting + +from .. import payments + + +PAYMENT_CURRENCY = Setting('PAYMENT_CURRENCY', + 'Eur' +) + + +PAYMENTS_DD_BILL_MESSAGE = Setting('PAYMENTS_DD_BILL_MESSAGE', + _("Direct debit, this bill will be automatically charged " + "to your bank account with IBAN number
    %(number)s."), +) + +PAYMENTS_DD_CREDITOR_NAME = Setting('PAYMENTS_DD_CREDITOR_NAME', + 'Orchestra' +) + + +PAYMENTS_DD_CREDITOR_IBAN = Setting('PAYMENTS_DD_CREDITOR_IBAN', + 'IE98BOFI90393912121212' +) + + +PAYMENTS_DD_CREDITOR_BIC = Setting('PAYMENTS_DD_CREDITOR_BIC', + 'BOFIIE2D' +) + + +PAYMENTS_DD_CREDITOR_AT02_ID = Setting('PAYMENTS_DD_CREDITOR_AT02_ID', + 'InvalidAT02ID' +) + + +PAYMENTS_ENABLED_METHODS = Setting('PAYMENTS_ENABLED_METHODS', + ( + 'orchestra.contrib.payments.methods.sepadirectdebit.SEPADirectDebit', + 'orchestra.contrib.payments.methods.creditcard.CreditCard', + ), + # lazy loading + choices=lambda : ((m.get_class_path(), m.get_class_path()) for m in payments.methods.PaymentMethod.get_plugins(all=True)), + multiple=True, +) diff --git a/orchestra/contrib/payments/templates/admin/payments/transaction/get_processes.html b/orchestra/contrib/payments/templates/admin/payments/transaction/get_processes.html new file mode 100644 index 00000000..d214ca6d --- /dev/null +++ b/orchestra/contrib/payments/templates/admin/payments/transaction/get_processes.html @@ -0,0 +1,19 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} +{% load i18n admin_urls utils %} + + +{% block content %} +

    {{ content_message }}

    +
      + {% for proc in processes %} +
    • Process #{{ proc.id }} + {% if proc.file %} + + {% endif %} + {% if proc.data %} +
      • Data: {{ proc.data }}
      + {% endif %} +
      • + {% endfor %} +
    +{% endblock %} diff --git a/orchestra/contrib/payments/templates/admin/payments/transaction/report.html b/orchestra/contrib/payments/templates/admin/payments/transaction/report.html new file mode 100644 index 00000000..185e1b70 --- /dev/null +++ b/orchestra/contrib/payments/templates/admin/payments/transaction/report.html @@ -0,0 +1,81 @@ +{% load i18n %} + + + + Transaction Report + + + + + + + + + + +{% for state, amount in states.items %} + + + + +{% endfor %} + + + + +
    {% trans "Summary" %}{% trans "Amount" %}
    {{ state }}{{ amount }}
    {% trans "TOTAL" %}{{ total }}
    + + + + + + + + + + + + + + +{% for transaction in transactions %} + + + + + + + + + + + +{% endfor %} +
    ID{% trans "Bill" %}{% trans "Contact" %}IBAN{% trans "Amount" %}{% trans "State" %}{% trans "Created" %}{% trans "Updated" %}
    {{ transaction.id }}{{ transaction.bill.number }}{{ transaction.bill.buyer.get_name }}{{ transaction.source.data.iban }}{{ transaction.amount }}{{ transaction.get_state_display }}{{ transaction.created_at|date }}{% if transaction.created_at|date != transaction.modified_at|date %}{{ transaction.modified_at|date }}{% else %} --- {% endif %}
    + + diff --git a/orchestra/contrib/plans/__init__.py b/orchestra/contrib/plans/__init__.py new file mode 100644 index 00000000..e09642bf --- /dev/null +++ b/orchestra/contrib/plans/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.plans.apps.PlansConfig' diff --git a/orchestra/contrib/plans/admin.py b/orchestra/contrib/plans/admin.py new file mode 100644 index 00000000..0a9ebacb --- /dev/null +++ b/orchestra/contrib/plans/admin.py @@ -0,0 +1,59 @@ +from django.contrib import admin +from django.urls import reverse +from django.db import models +from django.utils.html import format_html +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import insertattr, admin_link +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.contrib.services.models import Service + +from .models import Plan, ContractedPlan, Rate + + +class RateInline(admin.TabularInline): + model = Rate + ordering = ('service', 'plan', 'quantity') + + +class PlanAdmin(ExtendedModelAdmin): + list_display = ( + 'name', 'is_default', 'is_combinable', 'allow_multiple', 'is_active', 'num_contracts', + ) + list_filter = ('is_default', 'is_combinable', 'allow_multiple', 'is_active') + fields = ('verbose_name', 'name', 'is_default', 'is_combinable', 'allow_multiple') + prepopulated_fields = { + 'name': ('verbose_name',) + } + change_readonly_fields = ('name',) + inlines = [RateInline] + + def num_contracts(self, plan): + num = plan.contracts__count + url = reverse('admin:plans_contractedplan_changelist') + url += '?plan__name={}'.format(plan.name) + return format_html('{1}', url, num) + num_contracts.short_description = _("Contracts") + num_contracts.admin_order_field = 'contracts__count' + + def get_queryset(self, request): + qs = super(PlanAdmin, self).get_queryset(request) + return qs.annotate(models.Count('contracts', distinct=True)) + + +class ContractedPlanAdmin(AccountAdminMixin, admin.ModelAdmin): + list_display = ('id', 'plan_link', 'account_link') + list_filter = ('plan__name',) + list_select_related = ('plan', 'account') + search_fields = ('account__username', 'plan__name', 'id') + actions = (list_accounts,) + + plan_link = admin_link('plan') + + +admin.site.register(Plan, PlanAdmin) +admin.site.register(ContractedPlan, ContractedPlanAdmin) + +insertattr(Service, 'inlines', RateInline) diff --git a/orchestra/contrib/plans/apps.py b/orchestra/contrib/plans/apps.py new file mode 100644 index 00000000..45d30773 --- /dev/null +++ b/orchestra/contrib/plans/apps.py @@ -0,0 +1,16 @@ +from django.apps import AppConfig + +from orchestra.core import administration, accounts, services +from orchestra.core.translations import ModelTranslation + + +class PlansConfig(AppConfig): + name = 'orchestra.contrib.plans' + verbose_name = 'Plans' + + def ready(self): + from .models import Plan, ContractedPlan + accounts.register(ContractedPlan, icon='ContractedPack.png') + services.register(ContractedPlan, menu=False, dashboard=False) + administration.register(Plan, icon='Pack.png') + ModelTranslation.register(Plan, ('verbose_name',)) diff --git a/orchestra/contrib/plans/models.py b/orchestra/contrib/plans/models.py new file mode 100644 index 00000000..429ecc39 --- /dev/null +++ b/orchestra/contrib/plans/models.py @@ -0,0 +1,104 @@ +from functools import lru_cache + +from django.core.validators import ValidationError +from django.db import models +from django.db.models import Q +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from orchestra.core.validators import validate_name +from orchestra.models import queryset +from orchestra.utils.python import import_class + +from . import settings + + +class Plan(models.Model): + name = models.CharField(_("name"), max_length=32, unique=True, validators=[validate_name]) + verbose_name = models.CharField(_("verbose_name"), max_length=128, blank=True) + is_active = models.BooleanField(_("active"), default=True, + help_text=_("Designates whether this account should be treated as active. " + "Unselect this instead of deleting accounts.")) + is_default = models.BooleanField(_("default"), default=False, + help_text=_("Designates whether this plan is used by default or not.")) + is_combinable = models.BooleanField(_("combinable"), default=True, + help_text=_("Designates whether this plan can be combined with other plans or not.")) + allow_multiple = models.BooleanField(_("allow multiple"), default=False, + help_text=_("Designates whether this plan allow for multiple contractions.")) + + def __str__(self): + return self.get_verbose_name() + + def clean(self): + self.verbose_name = self.verbose_name.strip() + + def get_verbose_name(self): + return self.verbose_name or self.name + + +class ContractedPlan(models.Model): + plan = models.ForeignKey(Plan, on_delete=models.CASCADE, + verbose_name=_("plan"), related_name='contracts') + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='plans') + + class Meta: + verbose_name_plural = _("plans") + + def __str__(self): + return str(self.plan) + + @cached_property + def active(self): + return self.plan.is_active and self.account.is_active + + def clean(self): + if not self.pk and not self.plan.allow_multiple: + if ContractedPlan.objects.filter(plan=self.plan, account=self.account).exists(): + raise ValidationError("A contracted plan for this account already exists.") + + +class RateQuerySet(models.QuerySet): + group_by = queryset.group_by + + def by_account(self, account): + # Default allways selected + return self.filter( + Q(plan__is_default=True) | + Q(plan__contracts__account=account) + ).order_by('plan', 'quantity').select_related('plan', 'service') + + +class Rate(models.Model): + service = models.ForeignKey('services.Service', on_delete=models.CASCADE, + verbose_name=_("service"), related_name='rates') + plan = models.ForeignKey(Plan, on_delete=models.SET_NULL, null=True, blank=True, + verbose_name=_("plan"), related_name='rates') + quantity = models.PositiveIntegerField(_("quantity"), null=True, blank=True, + help_text=_("See rate algorihm help text.")) + price = models.DecimalField(_("price"), max_digits=12, decimal_places=2) + + objects = RateQuerySet.as_manager() + + class Meta: + unique_together = ('service', 'plan', 'quantity') + + def __str__(self): + return "{}-{}".format(str(self.price), self.quantity) + + @classmethod + @lru_cache() + def get_methods(cls): + return dict((method, import_class(method)) for method in settings.PLANS_RATE_METHODS) + + @classmethod + @lru_cache() + def get_choices(cls): + choices = [] + for name, method in cls.get_methods().items(): + choices.append((name, method.verbose_name)) + return choices + + @classmethod + def get_default(cls): + return settings.PLANS_DEFAULT_RATE_METHOD diff --git a/orchestra/contrib/plans/ratings.py b/orchestra/contrib/plans/ratings.py new file mode 100644 index 00000000..1c1cd9e3 --- /dev/null +++ b/orchestra/contrib/plans/ratings.py @@ -0,0 +1,212 @@ +import sys + +from django.utils.translation import gettext_lazy as _ + +from orchestra.utils.python import AttrDict + + +def _compute_steps(rates, metric): + value = 0 + num = len(rates) + accumulated = 0 + barrier = 1 + next_barrier = None + end = False + ix = 0 + steps = [] + while ix < num and not end: + fold = 1 + # Multiple contractions + while ix < num-1 and rates[ix] == rates[ix+1]: + ix += 1 + fold += 1 + if ix+1 == num: + quantity = metric - accumulated + next_barrier = quantity + else: + quantity = rates[ix+1].quantity - max(rates[ix].quantity, 1) + next_barrier = quantity + if rates[ix+1].price > rates[ix].price: + quantity *= fold + if accumulated+quantity > metric: + quantity = metric - accumulated + end = True + price = rates[ix].price + steps.append(AttrDict(**{ + 'quantity': quantity, + 'price': price, + 'barrier': barrier, + })) + accumulated += quantity + barrier += next_barrier + value += quantity*price + ix += 1 + return value, steps + + +def _standardize(rates): + """ + Support for incomplete rates + When first rate (quantity=5, price=10) defaults to nominal_price + """ + std_rates = [] + minimal = rates[0].quantity + for rate in rates: + #if rate.quantity == 0: + # rate.quantity = 1 + if rate.quantity == minimal and rate.quantity > 0: + service = rate.service + rate_class = type(rate) + std_rates.append( + rate_class(service=service, plan=rate.plan, quantity=0, price=service.nominal_price) + ) + std_rates.append(rate) + return std_rates + + +def step_price(rates, metric): + if rates.query.order_by != ('plan', 'quantity'): + raise ValueError("rates queryset should be ordered by 'plan' and 'quantity'") + # Step price + group = [] + minimal = (sys.maxsize, []) + for plan, rates in rates.group_by('plan').items(): + rates = _standardize(rates) + value, steps = _compute_steps(rates, metric) + if plan.is_combinable: + group.append(steps) + else: + minimal = min(minimal, (value, steps), key=lambda v: v[0]) + if len(group) == 1: + value, steps = _compute_steps(rates, metric) + minimal = min(minimal, (value, steps), key=lambda v: v[0]) + elif len(group) > 1: + # Merge + steps = [] + for rates in group: + steps += rates + steps.sort(key=lambda s: s.price) + result = [] + counter = 0 + value = 0 + ix = 0 + targets = [] + while counter < metric: + barrier = steps[ix].barrier + if barrier <= counter+1: + price = steps[ix].price + quantity = steps[ix].quantity + if quantity + counter > metric: + quantity = metric - counter + else: + for target in targets: + if counter + quantity >= target: + quantity = (counter+quantity+1) - target + steps[ix].quantity -= quantity + if not steps[ix].quantity: + steps.pop(ix) + break + else: + steps.pop(ix) + counter += quantity + value += quantity*price + if result and result[-1].price == price: + result[-1].quantity += quantity + else: + result.append(AttrDict(quantity=quantity, price=price)) + ix = 0 + targets = [] + else: + targets.append(barrier) + ix += 1 + minimal = min(minimal, (value, result), key=lambda v: v[0]) + return minimal[1] +step_price.verbose_name = _("Step price") +step_price.help_text = _("All rates with a quantity lower or equal than the metric are applied. " + "Nominal price will be used when initial block is missing.") + + +def match_price(rates, metric): + if rates.query.order_by != ('plan', 'quantity'): + raise ValueError("rates queryset should be ordered by 'plan' and 'quantity'") + candidates = [] + selected = False + prev = None + rates = _standardize(rates.distinct()) + for rate in rates: + if prev: + if prev.plan != rate.plan: + if not selected and prev.quantity <= metric: + candidates.append(prev) + selected = False + if not selected and rate.quantity > metric: + if prev.quantity <= metric: + candidates.append(prev) + selected = True + prev = rate + if not selected and prev.quantity <= metric: + candidates.append(prev) + candidates.sort(key=lambda r: r.price) + if candidates: + return [AttrDict(**{ + 'quantity': metric, + 'price': candidates[0].price, + })] + return None +match_price.verbose_name = _("Match price") +match_price.help_text = _("Only the rate with a) inmediate inferior metric and b) lower price is applied. " + "Nominal price will be used when initial block is missing.") + + +def best_price(rates, metric): + if rates.query.order_by != ('plan', 'quantity'): + raise ValueError("rates queryset should be ordered by 'plan' and 'quantity'") + candidates = [] + for plan, rates in rates.group_by('plan').items(): + rates = _standardize(rates) + plan_candidates = [] + for rate in rates: + if rate.quantity > metric: + break + if plan_candidates: + ant = plan_candidates[-1] + if ant.price == rate.price: + # Multiple plans support + ant.fold += 1 + else: + ant.quantity = rate.quantity-1 + plan_candidates.append(AttrDict( + price=rate.price, + quantity=metric, + fold=1, + )) + else: + plan_candidates.append(AttrDict( + price=rate.price, + quantity=metric, + fold=1, + )) + candidates.extend(plan_candidates) + results = [] + accumulated = 0 + for candidate in sorted(candidates, key=lambda c: c.price): + if candidate.quantity < accumulated: + # Out of barrier + continue + candidate.quantity *= candidate.fold + if accumulated+candidate.quantity > metric: + quantity = metric - accumulated + else: + quantity = candidate.quantity + accumulated += quantity + if quantity: + if results and results[-1].price == candidate.price: + results[-1].quantity += quantity + else: + results.append(AttrDict(**{ + 'quantity': quantity, + 'price': candidate.price + })) + return results +best_price.verbose_name = _("Best price") +best_price.help_text = _("Produces the best possible price given all active rating lines (those with quantity lower or equal to the metric).") diff --git a/orchestra/contrib/plans/settings.py b/orchestra/contrib/plans/settings.py new file mode 100644 index 00000000..853f6e48 --- /dev/null +++ b/orchestra/contrib/plans/settings.py @@ -0,0 +1,15 @@ +from orchestra.contrib.settings import Setting + + +PLANS_RATE_METHODS = Setting('PLANS_RATE_METHODS', + ( + 'orchestra.contrib.plans.ratings.step_price', + 'orchestra.contrib.plans.ratings.match_price', + 'orchestra.contrib.plans.ratings.best_price', + ) +) + + +PLANS_DEFAULT_RATE_METHOD = Setting('PLANS_DEFAULT_RATE_METHOD', + 'orchestra.contrib.plans.ratings.step_price', +) diff --git a/orchestra/contrib/resources/__init__.py b/orchestra/contrib/resources/__init__.py new file mode 100644 index 00000000..7c273a58 --- /dev/null +++ b/orchestra/contrib/resources/__init__.py @@ -0,0 +1,4 @@ +from .backends import ServiceMonitor + + +default_app_config = 'orchestra.contrib.resources.apps.ResourcesConfig' diff --git a/orchestra/contrib/resources/actions.py b/orchestra/contrib/resources/actions.py new file mode 100644 index 00000000..3120213b --- /dev/null +++ b/orchestra/contrib/resources/actions.py @@ -0,0 +1,47 @@ +from django.urls import reverse +from django.shortcuts import redirect, render +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext_lazy as _ + + +def run_monitor(modeladmin, request, queryset): + """ Resource and ResourceData run monitors """ + referer = request.META.get('HTTP_REFERER') + run_async = modeladmin.model.monitor.__defaults__[0] + logs = set() + for resource in queryset: + rlogs = resource.monitor() + if not run_async: + logs = logs.union(set([str(log.pk) for log in rlogs])) + modeladmin.log_change(request, resource, _("Run monitors")) + if run_async: + num = len(queryset) + # TODO listfilter by uuid: task.request.id + ?task_id__in=ids + link = reverse('admin:djcelery_taskstate_changelist') + msg = ngettext( + _("One selected resource has been scheduled for monitoring.") % link, + _("%s selected resource have been scheduled for monitoring.") % (num, link), + num) + else: + num = len(logs) + if num == 1: + log_pk = int(logs.pop()) + link = reverse('admin:orchestration_backendlog_change', args=(log_pk,)) + msg = _("One related monitor has been executed.") % link + elif num >= 1: + link = reverse('admin:orchestration_backendlog_changelist') + link += '?id__in=%s' % ','.join(logs) + msg = _("%s related monitors have been executed.") % (num, link) + else: + msg = _("No related monitors have been executed.") + modeladmin.message_user(request, mark_safe(msg)) + if referer: + return redirect(referer) +run_monitor.url_name = 'monitor' + + +def show_history(modeladmin, request, queryset): + context = { + 'ids': ','.join(map(str, queryset.values_list('id', flat=True))), + } + return render(request, 'admin/resources/resourcedata/history.html', context) diff --git a/orchestra/contrib/resources/admin.py b/orchestra/contrib/resources/admin.py new file mode 100644 index 00000000..53f573ee --- /dev/null +++ b/orchestra/contrib/resources/admin.py @@ -0,0 +1,356 @@ +from urllib.parse import parse_qs + +from django.apps import apps +from django.urls import re_path as url +from django.contrib import admin, messages +from django.contrib.contenttypes.admin import GenericTabularInline +from django.contrib.contenttypes.forms import BaseGenericInlineFormSet +from django.contrib.admin.utils import unquote +from django.urls import reverse +from django.db.models import Q +from django.shortcuts import redirect +from django.templatetags.static import static +from django.utils.functional import cached_property +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date +from orchestra.contrib.orchestration.models import Route +from orchestra.core import services +from orchestra.utils import db, sys +from orchestra.utils.functional import cached + +from .actions import run_monitor, show_history +from .api import history_data +from .filters import ResourceDataListFilter +from .forms import ResourceForm +from .models import Resource, ResourceData, MonitorData + + +class ResourceAdmin(ExtendedModelAdmin): + list_display = ( + 'id', 'verbose_name', 'content_type', 'aggregation', 'on_demand', + 'default_allocation', 'unit', 'crontab', 'is_active' + ) + list_display_links = ('id', 'verbose_name') + list_editable = ('default_allocation', 'crontab', 'is_active',) + list_filter = ( + ('content_type', admin.RelatedOnlyFieldListFilter), 'aggregation', 'on_demand', + 'disable_trigger' + ) + fieldsets = ( + (None, { + 'fields': ('verbose_name', 'name', 'content_type', 'aggregation'), + }), + (_("Configuration"), { + 'fields': ('unit', 'scale', 'on_demand', 'default_allocation', 'disable_trigger', + 'is_active'), + }), + (_("Monitoring"), { + 'fields': ('monitors', 'crontab'), + }), + ) + actions = (run_monitor,) + change_view_actions = actions + change_readonly_fields = ('name', 'content_type') + prepopulated_fields = { + 'name': ('verbose_name',) + } + list_select_related = ('content_type', 'crontab',) + + def change_view(self, request, object_id, form_url='', extra_context=None): + """ Remaind user when monitor routes are not configured """ + if request.method == 'GET': + resource = self.get_object(request, unquote(object_id)) + backends = Route.objects.values_list('backend', flat=True) + not_routed = [] + for monitor in resource.monitors: + if monitor not in backends: + not_routed.append(monitor) + if not_routed: + messages.warning(request, ngettext( + _("%(not_routed)s monitor doesn't have any configured route."), + _("%(not_routed)s monitors don't have any configured route."), + len(not_routed), + ) % { + 'not_routed': ', '.join(not_routed) + }) + return super(ResourceAdmin, self).change_view(request, object_id, form_url=form_url, + extra_context=extra_context) + + def save_model(self, request, obj, form, change): + super(ResourceAdmin, self).save_model(request, obj, form, change) + # best-effort + model = obj.content_type.model_class() + modeladmin = type(get_modeladmin(model)) + resources = obj.content_type.resource_set.filter(is_active=True) + inlines = [] + for inline in modeladmin.inlines: + if inline.model is ResourceData: + inline = resource_inline_factory(resources) + inlines.append(inline) + modeladmin.inlines = inlines + # reload Not always work + sys.touch_wsgi() + + def formfield_for_dbfield(self, db_field, **kwargs): + """ filter service content_types """ + if db_field.name == 'content_type': + models = [ model._meta.model_name for model in services.get() ] + kwargs['queryset'] = db_field.remote_field.model.objects.filter(model__in=models) + return super(ResourceAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + +def content_object_link(data): + ct = data.content_type + url = reverse('admin:%s_%s_change' % (ct.app_label, ct.model), args=(data.object_id,)) + return format_html('{}', url, data.content_object_repr) +content_object_link.short_description = _("Content object") +content_object_link.admin_order_field = 'content_object_repr' + + +class ResourceDataAdmin(ExtendedModelAdmin): + list_display = ( + 'id', 'resource_link', content_object_link, 'allocated', 'display_used', + 'display_updated' + ) + list_filter = ('resource',) + fields = ( + 'resource_link', 'content_type', content_object_link, 'display_updated', 'display_used', + 'allocated', + ) + search_fields = ('content_object_repr',) + readonly_fields = fields + actions = (run_monitor, show_history) + change_view_actions = actions + ordering = ('-updated_at',) + list_select_related = ('resource__content_type', 'content_type') + + resource_link = admin_link('resource') + display_updated = admin_date('updated_at', short_description=_("Updated")) + + def get_urls(self): + """Returns the additional urls for the change view links""" + urls = super(ResourceDataAdmin, self).get_urls() + admin_site = self.admin_site + opts = self.model._meta + return [ + url('^(\d+)/used-monitordata/$', + admin_site.admin_view(self.used_monitordata_view), + name='%s_%s_used_monitordata' % (opts.app_label, opts.model_name) + ), + url('^history_data/$', + admin_site.admin_view(history_data), + name='%s_%s_history_data' % (opts.app_label, opts.model_name) + ), + url('^list-related/(.+)/(.+)/(\d+)/$', + admin_site.admin_view(self.list_related_view), + name='%s_%s_list_related' % (opts.app_label, opts.model_name) + ), + ] + urls + + def display_used(self, rdata): + if rdata.used is None: + return '' + url = reverse('admin:resources_resourcedata_used_monitordata', args=(rdata.pk,)) + return format_html('{} {}', url, rdata.used, rdata.unit) + display_used.short_description = _("Used") + display_used.admin_order_field = 'used' + + def has_add_permission(self, *args, **kwargs): + return False + + def used_monitordata_view(self, request, object_id): + url = reverse('admin:resources_monitordata_changelist') + url += '?resource_data=%s' % object_id + return redirect(url) + + def list_related_view(self, request, app_name, model_name, object_id): + resources = Resource.objects.select_related('content_type') + resource_models = {r.content_type.model_class(): r.content_type_id for r in resources} + # Self + model = apps.get_model(app_name, model_name) + obj = model.objects.get(id=int(object_id)) + ct_id = resource_models[model] + qset = Q(content_type_id=ct_id, object_id=obj.id, resource__is_active=True) + # Related + for field, rel in obj._meta.fields_map.items(): + try: + ct_id = resource_models[rel.related_model] + except KeyError: + pass + else: + manager = getattr(obj, field) + ids = manager.values_list('id', flat=True) + qset = Q(qset) | Q(content_type_id=ct_id, object_id__in=ids, resource__is_active=True) + related = ResourceData.objects.filter(qset) + related_ids = related.values_list('id', flat=True) + related_ids = ','.join(map(str, related_ids)) + url = reverse('admin:resources_resourcedata_changelist') + url += '?id__in=%s' % related_ids + return redirect(url) + + +class MonitorDataAdmin(ExtendedModelAdmin): + list_display = ('id', 'monitor', content_object_link, 'display_created', 'value') + list_filter = ('monitor', ResourceDataListFilter) + add_fields = ('monitor', 'content_type', 'object_id', 'created_at', 'value') + fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value', 'state') + change_readonly_fields = fields + list_select_related = ('content_type',) + search_fields = ('content_object_repr',) + date_hierarchy = 'created_at' + + display_created = admin_date('created_at', short_description=_("Created")) + + def filter_used_monitordata(self, request, queryset): + query_string = parse_qs(request.META['QUERY_STRING']) + resource_data = query_string.get('resource_data') + if resource_data: + mdata = ResourceData.objects.get(pk=int(resource_data[0])) + resource = mdata.resource + ids = [] + for monitor, dataset in mdata.get_monitor_datasets(): + dataset = resource.aggregation_instance.filter(dataset) + if isinstance(dataset, MonitorData): + ids.append(dataset.id) + else: + ids += dataset.values_list('id', flat=True) + return queryset.filter(id__in=ids) + return queryset + + def get_queryset(self, request): + queryset = super(MonitorDataAdmin, self).get_queryset(request) + queryset = self.filter_used_monitordata(request, queryset) + return queryset.prefetch_related('content_object') + + +admin.site.register(Resource, ResourceAdmin) +admin.site.register(ResourceData, ResourceDataAdmin) +admin.site.register(MonitorData, MonitorDataAdmin) + + +# Mokey-patching + +def resource_inline_factory(resources): + class ResourceInlineFormSet(BaseGenericInlineFormSet): + def total_form_count(self, resources=resources): + return len(resources) + + @cached + def get_queryset(self): + """ Filter disabled resources """ + queryset = super(ResourceInlineFormSet, self).get_queryset() + return queryset.filter(resource__is_active=True).select_related('resource') + + @cached_property + def forms(self, resources=resources): + forms = [] + resources_copy = list(resources) + # Remove queryset disabled objects + queryset = [rdata for rdata in self.get_queryset() if rdata.resource in resources] + if self.instance.pk: + # Create missing resource data + queryset_resources = [rdata.resource for rdata in queryset] + for resource in resources: + if resource not in queryset_resources: + kwargs = { + 'content_object': self.instance, + 'content_object_repr': str(self.instance), + } + if resource.default_allocation: + kwargs['allocated'] = resource.default_allocation + rdata = resource.dataset.create(**kwargs) + queryset.append(rdata) + # Existing dataset + for i, rdata in enumerate(queryset): + forms.append(self._construct_form(i, resource=rdata.resource)) + try: + resources_copy.remove(rdata.resource) + except ValueError: + pass + # Missing dataset + for i, resource in enumerate(resources_copy, len(queryset)): + forms.append(self._construct_form(i, resource=resource)) + return forms + + class ResourceInline(GenericTabularInline): + model = ResourceData + verbose_name_plural = _("resources") + form = ResourceForm + formset = ResourceInlineFormSet + can_delete = False + fields = ( + 'verbose_name', 'display_used', 'display_updated', 'allocated', 'unit', + ) + readonly_fields = ('display_used', 'display_updated',) + + class Media: + css = { + 'all': ('orchestra/css/hide-inline-id.css',) + } + + display_updated = admin_date('updated_at', default=_("Never")) + + def get_fieldsets(self, request, obj=None): + if obj: + opts = self.parent_model._meta + url = reverse('admin:resources_resourcedata_list_related', + args=(opts.app_label, opts.model_name, obj.id)) + link = '%s' % (url, _("List related")) + self.verbose_name_plural = mark_safe(_("Resources") + ' ' + link) + return super(ResourceInline, self).get_fieldsets(request, obj) + + @mark_safe + def display_used(self, rdata): + update = '' + history = '' + if rdata.pk: + context = { + 'title': _("Update"), + 'url': reverse('admin:resources_resourcedata_monitor', args=(rdata.pk,)), + 'image': '' % static('orchestra/images/reload.png'), + } + update = '%(image)s' % context + context.update({ + 'title': _("Show history"), + 'image': '' % static('orchestra/images/history.png'), + 'url': reverse('admin:resources_resourcedata_show_history', args=(rdata.pk,)), + 'popup': 'onclick="return showAddAnotherPopup(this);"', + }) + history = '%(image)s' % context + if rdata.used is not None: + used_url = reverse('admin:resources_resourcedata_used_monitordata', args=(rdata.pk,)) + used = '%s %s' % (used_url, rdata.used, rdata.unit) + return ' '.join(map(str, (used, update, history))) + if rdata.resource.monitors: + return _("Unknonw %s %s") % (update, history) + return _("No monitor") + display_used.short_description = _("Used") + + def has_add_permission(self, *args, **kwargs): + """ Hidde add another """ + return False + + return ResourceInline + + +def insert_resource_inlines(): + # Clean previous state + for related in Resource._related: + modeladmin = get_modeladmin(related) + modeladmin_class = type(modeladmin) + for inline in getattr(modeladmin_class, 'inlines', []): + if inline.__name__ == 'ResourceInline': + modeladmin_class.inlines.remove(inline) + resources = Resource.objects.filter(is_active=True) + for ct, resources in resources.group_by('content_type').items(): + inline = resource_inline_factory(resources) + model = ct.model_class() + insertattr(model, 'inlines', inline) + + +if db.database_ready(): + insert_resource_inlines() diff --git a/orchestra/contrib/resources/aggregations.py b/orchestra/contrib/resources/aggregations.py new file mode 100644 index 00000000..f43e7ffd --- /dev/null +++ b/orchestra/contrib/resources/aggregations.py @@ -0,0 +1,169 @@ +import datetime +import decimal +import itertools + +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from orchestra.utils.python import AttrDict + +from orchestra import plugins + + +class Aggregation(plugins.Plugin, metaclass=plugins.PluginMount): + """ filters and computes dataset usage """ + aggregated_history = False + + def filter(self, dataset): + """ Filter the dataset to get the relevant data according to the period """ + raise NotImplementedError + + def compute_usage(self, dataset): + """ given a dataset computes its usage according to the method (avg, sum, ...) """ + raise NotImplementedError + + def aggregate_history(self, dataset): + raise NotImplementedError + + +class Last(Aggregation): + """ Sum of the last value of all monitors """ + name = 'last' + verbose_name = _("Last value") + + def filter(self, dataset, date=None): + dataset = dataset.order_by('object_id', '-id').distinct('monitor') + if date is not None: + dataset = dataset.filter(created_at__lte=date) + return dataset + + def compute_usage(self, dataset): + values = dataset.values_list('value', flat=True) + if values: + return sum(values) + return None + + def aggregate_history(self, dataset): + prev_object_id = None + prev_object_repr = None + for mdata in dataset.order_by('object_id', 'created_at'): + object_id = mdata.object_id + if object_id != prev_object_id: + if prev_object_id is not None: + yield (prev_object_repr, datas) + datas = [mdata] + else: + datas.append(mdata) + prev_object_id = object_id + prev_object_repr = mdata.content_object_repr + if prev_object_id is not None: + yield (prev_object_repr, datas) + + +class MonthlySum(Last): + """ Monthly sum the values of all monitors """ + name = 'monthly-sum' + verbose_name = _("Monthly Sum") + aggregated_history = True + + def filter(self, dataset, date=None): + if date is None: + date = timezone.now().date() + return dataset.filter( + created_at__year=date.year, + created_at__month=date.month, + ) + + def aggregate_history(self, dataset): + prev = None + prev_object_id = None + datas = [] + sink = AttrDict(object_id=-1, value=-1, content_object_repr='', + created_at=AttrDict(year=-1, month=-1)) + for mdata in itertools.chain(dataset.order_by('object_id', 'created_at'), [sink]): + object_id = mdata.object_id + ymonth = (mdata.created_at.year, mdata.created_at.month) + if object_id != prev_object_id or ymonth != prev.ymonth: + if prev_object_id is not None: + data = AttrDict( + date=datetime.date( + year=prev.ymonth[0], + month=prev.ymonth[1], + day=1 + ), + value=current, + content_object_repr=prev.content_object_repr + ) + datas.append(data) + current = mdata.value + else: + current += mdata.value + if object_id != prev_object_id: + if prev_object_id is not None: + yield (prev.content_object_repr, datas) + datas = [] + prev = mdata + prev.ymonth = ymonth + prev_object_id = object_id + + +class MonthlyAvg(MonthlySum): + """ sum of the monthly averages of each monitor """ + name = 'monthly-avg' + verbose_name = _("Monthly AVG") + aggregated_history = False + + def get_epoch(self, date=None): + if date is None: + date = timezone.now().date() + return datetime.date( + year=date.year, + month=date.month, + day=1, + ) + + def compute_usage(self, dataset): + result = 0 + has_result = False + aggregate = [] + for object_id, dataset in dataset.order_by('created_at').group_by('object_id').items(): + try: + last = dataset[-1] + except IndexError: + continue + epoch = self.get_epoch(date=last.created_at) + total = (last.created_at-epoch).total_seconds() + ini = epoch + current = 0 + for mdata in dataset: + has_result = True + slot = (mdata.created_at-ini).total_seconds() + current += mdata.value * decimal.Decimal(str(slot/total)) + ini = mdata.created_at + else: + result += current + if has_result: + return result + return None + + def aggregate_history(self, dataset): + yield from super(MonthlySum, self).aggregate_history(dataset) + + +class Last10DaysAvg(MonthlyAvg): + """ sum of the last 10 days averages of each monitor """ + name = 'last-10-days-avg' + verbose_name = _("Last 10 days AVG") + days = 10 + + def get_epoch(self, date=None): + if date is None: + date = timezone.now().date() + return date - datetime.timedelta(days=self.days) + + def filter(self, dataset, date=None): + epoch = self.get_epoch(date=date) + dataset = dataset.filter(created_at__gt=epoch) + if date is not None: + dataset = dataset.filter(created_at__lte=date) + return dataset diff --git a/orchestra/contrib/resources/api.py b/orchestra/contrib/resources/api.py new file mode 100644 index 00000000..9f89fb1e --- /dev/null +++ b/orchestra/contrib/resources/api.py @@ -0,0 +1,15 @@ +import json +from urllib.parse import parse_qs + +from django.http import HttpResponse + +from .helpers import get_history_data +from .models import ResourceData + + +def history_data(request): + ids = map(int, parse_qs(request.META['QUERY_STRING'])['ids'][0].split(',')) + queryset = ResourceData.objects.filter(id__in=ids) + history = get_history_data(queryset) + response = json.dumps(history, indent=4) + return HttpResponse(response, content_type="application/json") diff --git a/orchestra/contrib/resources/apps.py b/orchestra/contrib/resources/apps.py new file mode 100644 index 00000000..1b0c90ed --- /dev/null +++ b/orchestra/contrib/resources/apps.py @@ -0,0 +1,32 @@ +from django import db +from django.apps import AppConfig + +from orchestra.core import administration +from orchestra.utils.db import database_ready + + +class ResourcesConfig(AppConfig): + name = 'orchestra.contrib.resources' + verbose_name = 'Resources' + + def ready(self): + if database_ready(): + from .models import create_resource_relation + try: + create_resource_relation() + except db.utils.OperationalError: + # Not ready afterall + pass + from .models import Resource, ResourceData, MonitorData + administration.register(Resource, icon='gauge.png') + administration.register(ResourceData, parent=Resource, icon='monitor.png') + administration.register(MonitorData, parent=Resource, dashboard=False) + from . import signals + + def reload_relations(self): + from .admin import insert_resource_inlines + from .models import create_resource_relation + from .serializers import insert_resource_serializers + insert_resource_inlines() + insert_resource_serializers() + create_resource_relation() diff --git a/orchestra/contrib/resources/backends.py b/orchestra/contrib/resources/backends.py new file mode 100644 index 00000000..a997df5f --- /dev/null +++ b/orchestra/contrib/resources/backends.py @@ -0,0 +1,100 @@ +import datetime + +from django.utils import timezone +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceBackend + +from . import helpers + + +class ServiceMonitor(ServiceBackend): + TRAFFIC = 'traffic' + DISK = 'disk' + MEMORY = 'memory' + CPU = 'cpu' + # TODO UNITS + actions = ('monitor', 'exceeded', 'recovery') + abstract = True + delete_old_equal_values = False + monthly_sum_old_values = False + + @classmethod + def get_plugins(cls): + """ filter controller classes """ + return [ + plugin for plugin in cls.plugins if issubclass(plugin, ServiceMonitor) + ] + + @classmethod + def get_verbose_name(cls): + return _("[M] %s") % super(ServiceMonitor, cls).get_verbose_name() + + @cached_property + def current_date(self): + return timezone.now() + + @cached_property + def content_type(self): + from django.contrib.contenttypes.models import ContentType + app_label, model = self.model.split('.') + model = model.lower() + return ContentType.objects.get_by_natural_key(app_label, model) + + def get_last_data(self, object_id): + from .models import MonitorData + try: + return MonitorData.objects.filter(content_type=self.content_type, + monitor=self.get_name(), object_id=object_id).latest() + except MonitorData.DoesNotExist: + return None + + def get_last_date(self, object_id): + data = self.get_last_data(object_id) + if data is None: + return self.current_date - datetime.timedelta(days=1) + return data.created_at + + def process(self, line): + """ line -> object_id, value, state""" + result = line.split() + if len(result) != 2: + cls_name = self.__class__.__name__ + raise ValueError("%s expected ' ' got '%s'" % (cls_name, line)) + # State is None, unless your monitor needs to keep track of it + result.append(None) + return result + + def store(self, log): + """ stores monitored values from stdout """ + from django.contrib.contenttypes.models import ContentType + from .models import MonitorData + name = self.get_name() + app_label, model_name = self.model.split('.') + ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower()) + for line in log.stdout.splitlines(): + line = line.strip() + object_id, value, state = self.process(line) + if isinstance(value, bytes): + value = value.decode('ascii') + if isinstance(state, bytes): + state = state.decode('ascii') + content_object = ct.get_object_for_this_type(pk=object_id) + MonitorData.objects.create( + monitor=name, object_id=object_id, content_type=ct, value=value, state=state, + created_at=self.current_date, content_object_repr=str(content_object), + ) + + def execute(self, *args, **kwargs): + log = super(ServiceMonitor, self).execute(*args, **kwargs) + if log.state == log.SUCCESS: + self.store(log) + return log + + @classmethod + def aggregate(cls, dataset): + if cls.delete_old_equal_values: + return helpers.delete_old_equal_values(dataset) + elif cls.monthly_sum_old_values: + return helpers.monthly_sum_old_values(dataset) diff --git a/orchestra/contrib/resources/filters.py b/orchestra/contrib/resources/filters.py new file mode 100644 index 00000000..fe8392cd --- /dev/null +++ b/orchestra/contrib/resources/filters.py @@ -0,0 +1,17 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + + +class ResourceDataListFilter(SimpleListFilter): + """ Mock filter to avoid e=1 """ + title = _("Resource data") + parameter_name = 'resource_data' + + def lookups(self, request, model_admin): + return () + + def queryset(self, request, queryset): + return queryset + + def choices(self, cl): + return [] diff --git a/orchestra/contrib/resources/forms.py b/orchestra/contrib/resources/forms.py new file mode 100644 index 00000000..75d5b85a --- /dev/null +++ b/orchestra/contrib/resources/forms.py @@ -0,0 +1,44 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +from orchestra.forms import ReadOnlyFormMixin +from orchestra.forms.widgets import SpanWidget + + +class ResourceForm(ReadOnlyFormMixin, forms.ModelForm): + verbose_name = forms.CharField(label=_("Name"), required=False, + widget=SpanWidget(tag='')) + allocated = forms.DecimalField(label=_("Allocated")) + unit = forms.CharField(label=_("Unit"), required=False) + + class Meta: + fields = ('verbose_name', 'used', 'last_update', 'allocated', 'unit') + readonly_fields = ('verbose_name', 'unit') + + def __init__(self, *args, **kwargs): + self.resource = kwargs.pop('resource', None) + if self.resource: + initial = kwargs.get('initial', {}) + initial.update({ + 'verbose_name': self.resource.get_verbose_name(), + 'unit': self.resource.unit, + }) + kwargs['initial'] = initial + super(ResourceForm, self).__init__(*args, **kwargs) + if self.resource: + if self.resource.on_demand: + self.fields['allocated'].required = False + self.fields['allocated'].widget = SpanWidget(original=None, display='') + else: + self.fields['allocated'].required = True + self.fields['allocated'].initial = self.resource.default_allocation + +# def has_changed(self): +# """ Make sure resourcedata objects are created for all resources """ +# if not self.instance.pk: +# return True +# return super(ResourceForm, self).has_changed() + + def save(self, *args, **kwargs): + self.instance.resource_id = self.resource.pk + return super(ResourceForm, self).save(*args, **kwargs) diff --git a/orchestra/contrib/resources/helpers.py b/orchestra/contrib/resources/helpers.py new file mode 100644 index 00000000..d0d49873 --- /dev/null +++ b/orchestra/contrib/resources/helpers.py @@ -0,0 +1,134 @@ +import decimal + +from django.template.defaultfilters import date as date_format + + +def get_history_data(queryset): + resources = {} + needs_aggregation = False + for rdata in queryset: + resource = rdata.resource + try: + (options, aggregation) = resources[resource] + except KeyError: + aggregation = resource.aggregation_instance + options = { + 'aggregation': str(aggregation.verbose_name), + 'aggregated_history': aggregation.aggregated_history, + 'content_type': rdata.content_type.model, + 'content_object': rdata.content_object_repr, + 'unit': resource.unit, + 'scale': resource.get_scale(), + 'verbose_name': str(resource.verbose_name), + 'dates': set() if aggregation.aggregated_history else None, + 'objects': [], + } + resources[resource] = (options, aggregation) + if aggregation.aggregated_history: + needs_aggregation = True + monitors = [] + scale = options['scale'] + all_dates = options['dates'] + for monitor_name, dataset in rdata.get_monitor_datasets(): + datasets = {} + for content_object, datas in aggregation.aggregate_history(dataset): + if aggregation.aggregated_history: + serie = {} + for data in datas: + value = round(float(data.value)/scale, 3) if data.value is not None else None + all_dates.add(data.date) + serie[data.date] = value + else: + serie = [] + for data in datas: + date = data.created_at.timestamp() + date = int(str(date).split('.')[0] + '000') + value = round(float(data.value)/scale, 3) if data.value is not None else None + serie.append( + (date, value) + ) + datasets[content_object] = serie + monitors.append({ + 'name': monitor_name, + 'datasets': datasets, + }) + options['objects'].append({ + 'object_name': rdata.content_object_repr, + 'current': round(float(rdata.used or 0), 3), + 'allocated': float(rdata.allocated) if rdata.allocated is not None else None, + 'updated_at': rdata.updated_at.isoformat() if rdata.updated_at else None, + 'monitors': monitors, + }) + if needs_aggregation: + result = [] + for options, aggregation in resources.values(): + if aggregation.aggregated_history: + all_dates = sorted(options['dates']) + options['dates'] = [date_format(date) for date in all_dates] + for obj in options['objects']: + for monitor in obj['monitors']: + series = [] + for content_object, dataset in monitor['datasets'].items(): + data = [] + for date in all_dates: + data.append(dataset.get(date, 0.0)) + series.append({ + 'name': content_object, + 'data': data, + }) + monitor['datasets'] = series + result.append(options) + else: + result = [resource[0] for resource in resources.values()] + return result + + +def delete_old_equal_values(dataset): + """ only first and last values of an equal serie (+-error) are kept """ + prev_value = None + prev_key = None + delete_count = 0 + error = decimal.Decimal('0.005') + third = False + for mdata in dataset.order_by('content_type_id', 'object_id', 'created_at'): + key = (mdata.content_type_id, mdata.object_id) + if prev_key == key: + if prev_value is not None and mdata.value*(1-error) < prev_value < mdata.value*(1+error): + if third: + prev.delete() + delete_count += 1 + else: + third = True + else: + third = False + prev_value = mdata.value + prev_key = key + else: + prev_value = None + prev_key = key + prev = mdata + return delete_count + + +def monthly_sum_old_values(dataset): + aggregated = 0 + prev_key = None + prev = None + to_delete = [] + delete_count = 0 + for mdata in dataset.order_by('content_type_id', 'object_id', 'created_at'): + key = (mdata.content_type_id, mdata.object_id, mdata.created_at.year, mdata.created_at.month) + if prev_key is not None and prev_key != key: + if prev.value != aggregated: + prev.value = aggregated + prev.save(update_fields=('value',)) + for obj in to_delete[:-1]: + obj.delete() + delete_count += 1 + aggregated = 0 + to_delete = [] + prev = mdata + prev_key = key + aggregated += mdata.value + to_delete.append(mdata) + return delete_count diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py new file mode 100644 index 00000000..7b1e0ff9 --- /dev/null +++ b/orchestra/contrib/resources/models.py @@ -0,0 +1,353 @@ +from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation +from django.contrib.contenttypes.models import ContentType +from django.apps import apps +from django.db import models +from django.utils import timezone +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ +from djcelery.models import PeriodicTask + +from orchestra.core import validators +from orchestra.models import queryset, fields +from orchestra.models.utils import get_model_field_path + +from . import tasks +from .backends import ServiceMonitor +from .aggregations import Aggregation +from .validators import validate_scale + + +class ResourceQuerySet(models.QuerySet): + group_by = queryset.group_by + + +class Resource(models.Model): + """ + Defines a resource, a resource is basically an interpretation of data + gathered by a Monitor + """ + + LAST = 'LAST' + MONTHLY_SUM = 'MONTHLY_SUM' + MONTHLY_AVG = 'MONTHLY_AVG' + PERIODS = ( + (LAST, _("Last")), + (MONTHLY_SUM, _("Monthly sum")), + (MONTHLY_AVG, _("Monthly avg")), + ) + _related = set() # keeps track of related models for resource cleanup + + name = models.CharField(_("name"), max_length=32, + help_text=_("Required. 32 characters or fewer. Lowercase letters, " + "digits and hyphen only."), + validators=[validators.validate_name]) + verbose_name = models.CharField(_("verbose name"), max_length=256) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, + help_text=_("Model where this resource will be hooked.")) + aggregation = models.CharField(_("aggregation"), max_length=16, + choices=Aggregation.get_choices(), default=Aggregation.get_choices()[0][0], + help_text=_("Method used for aggregating this resource monitored data.")) + on_demand = models.BooleanField(_("on demand"), default=False, + help_text=_("If enabled the resource will not be pre-allocated, " + "but allocated under the application demand")) + default_allocation = models.PositiveIntegerField(_("default allocation"), + null=True, blank=True, + help_text=_("Default allocation value used when this is not an " + "on demand resource")) + unit = models.CharField(_("unit"), max_length=16, + help_text=_("The unit in which this resource is represented. " + "For example GB, KB or subscribers")) + scale = models.CharField(_("scale"), max_length=32, validators=[validate_scale], + help_text=_("Scale in which this resource monitoring resoults should " + "be prorcessed to match with unit. e.g. 10**9")) + disable_trigger = models.BooleanField(_("disable trigger"), default=True, + help_text=_("Disables monitors exeeded and recovery triggers")) + crontab = models.ForeignKey('djcelery.CrontabSchedule', verbose_name=_("crontab"), + null=True, blank=True, on_delete=models.SET_NULL, + help_text=_("Crontab for periodic execution. " + "Leave it empty to disable periodic monitoring")) + monitors = fields.MultiSelectField(_("monitors"), max_length=256, blank=True, + choices=ServiceMonitor.get_choices(), + help_text=_("Monitor backends used for monitoring this resource.")) + is_active = models.BooleanField(_("active"), default=True) + + objects = ResourceQuerySet.as_manager() + + class Meta: + unique_together = ( + ('name', 'content_type'), + ('verbose_name', 'content_type') + ) + + def __str__(self): + return "%s-%s" % (self.content_type, self.name) + + @cached_property + def aggregation_class(self): + return Aggregation.get(self.aggregation) + + @cached_property + def aggregation_instance(self): + """ Per request lived type_instance """ + return self.aggregation_class(self) + + def clean(self): + self.verbose_name = self.verbose_name.strip() + if self.on_demand and self.default_allocation: + raise validators.ValidationError({ + 'default_allocation': _("Default allocation can not be set for 'on demand' services") + }) + # Validate that model path exists between ct and each monitor.model + monitor_errors = [] + for monitor in self.monitors: + try: + self.get_model_path(monitor) + except (RuntimeError, LookupError): + model = apps.get_model(ServiceMonitor.get_backend(monitor).model) + monitor_errors.append(model._meta.model_name) + if monitor_errors: + model_name = self.content_type.model_class()._meta.model_name + raise validators.ValidationError({ + 'monitors': [ + _("Path does not exists between '%s' and '%s'") % ( + error, + model_name, + ) for error in monitor_errors + ]}) + + def save(self, *args, **kwargs): + super(Resource, self).save(*args, **kwargs) + # This only works on tests (multiprocessing used on real deployments) + apps.get_app_config('resources').reload_relations() + + def sync_periodic_task(self, delete=False): + """ sync periodic task on save/delete resource operations """ + name = 'monitor.%s' % self + if delete or not self.crontab or not self.is_active: + PeriodicTask.objects.filter(name=name).delete() + elif self.pk: + try: + task = PeriodicTask.objects.get(name=name) + except PeriodicTask.DoesNotExist: + if self.is_active: + PeriodicTask.objects.create( + name=name, + task='resources.Monitor', + args=[self.pk], + crontab=self.crontab + ) + else: + if task.crontab != self.crontab: + task.crontab = self.crontab + task.save(update_fields=['crontab']) + + def get_model_path(self, monitor): + """ returns a model path between self.content_type and monitor.model """ + resource_model = self.content_type.model_class() + monitor_model = ServiceMonitor.get_backend(monitor).model_class() + return get_model_field_path(monitor_model, resource_model) + + def get_scale(self): + return eval(self.scale) + + def get_verbose_name(self): + return self.verbose_name or self.name + + def monitor(self, run_async=True): + if run_async: + return tasks.monitor.apply_async(self.pk) + return tasks.monitor(self.pk) + + +class ResourceDataQuerySet(models.QuerySet): + def get_or_create(self, obj, resource): + ct = ContentType.objects.get_for_model(type(obj)) + try: + return self.get( + content_type=ct, + object_id=obj.pk, + resource=resource + ), False + except self.model.DoesNotExist: + return self.create( + content_object=obj, + resource=resource, + allocated=resource.default_allocation + ), True + + +class ResourceData(models.Model): + """ Stores computed resource usage and allocation """ + resource = models.ForeignKey(Resource, on_delete=models.CASCADE, related_name='dataset', verbose_name=_("resource")) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, verbose_name=_("content type")) + object_id = models.PositiveIntegerField(_("object id")) + used = models.DecimalField(_("used"), max_digits=16, decimal_places=3, null=True, + editable=False) + updated_at = models.DateTimeField(_("updated"), null=True, editable=False) + allocated = models.PositiveIntegerField(_("allocated"), null=True, blank=True) + content_object_repr = models.CharField(_("content object representation"), max_length=256, + editable=False) + + content_object = GenericForeignKey() + objects = ResourceDataQuerySet.as_manager() + + class Meta: + unique_together = ('resource', 'content_type', 'object_id') + verbose_name_plural = _("resource data") + index_together = ( + ('content_type', 'object_id'), + ) + + def __str__(self): + return "%s: %s" % (self.resource, self.content_object) + + @property + def unit(self): + return self.resource.unit + + @property + def verbose_name(self): + return self.resource.verbose_name + + def get_used(self): + resource = self.resource + total = 0 + has_result = False + for monitor, dataset in self.get_monitor_datasets(): + dataset = resource.aggregation_instance.filter(dataset) + usage = resource.aggregation_instance.compute_usage(dataset) + if usage is not None: + has_result = True + total += usage + return float(total)/resource.get_scale() if has_result else None + + def update(self, current=None): + if current is None: + current = self.get_used() + self.used = current or 0 + self.updated_at = timezone.now() + self.content_object_repr = str(self.content_object) + self.save(update_fields=('used', 'updated_at', 'content_object_repr')) + + def monitor(self, run_async=False): + ids = (self.object_id,) + if run_async: + return tasks.monitor.delay(self.resource_id, ids=ids) + return tasks.monitor(self.resource_id, ids=ids) + + def get_monitor_datasets(self): + resource = self.resource + for monitor in resource.monitors: + path = resource.get_model_path(monitor) + if path == []: + dataset = MonitorData.objects.filter( + monitor=monitor, + content_type=self.content_type_id, + object_id=self.object_id, + ) + else: + fields = '__'.join(path) + monitor_model = ServiceMonitor.get_backend(monitor).model_class() + objects = monitor_model.objects.filter(**{fields: self.object_id}) + pks = objects.values_list('id', flat=True) + ct = ContentType.objects.get_for_model(monitor_model) + dataset = MonitorData.objects.filter( + monitor=monitor, + content_type=ct, + object_id__in=pks, + ) + yield monitor, dataset + + +class MonitorDataQuerySet(models.QuerySet): + group_by = queryset.group_by + + +class MonitorData(models.Model): + """ Stores monitored data """ + monitor = models.CharField(_("monitor"), max_length=256, db_index=True, + choices=ServiceMonitor.get_choices()) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, verbose_name=_("content type")) + object_id = models.PositiveIntegerField(_("object id")) + created_at = models.DateTimeField(_("created"), default=timezone.now, db_index=True) + value = models.DecimalField(_("value"), max_digits=16, decimal_places=2) + state = models.DecimalField(_("state"), max_digits=16, decimal_places=2, null=True, + help_text=_("Optional field used to store current state needed for diff-based monitoring.")) + content_object_repr = models.CharField(_("content object representation"), max_length=256, + editable=False) + + content_object = GenericForeignKey() + objects = MonitorDataQuerySet.as_manager() + + class Meta: + get_latest_by = 'id' + verbose_name_plural = _("monitor data") + index_together = ( + ('content_type', 'object_id'), + ) + + def __str__(self): + return str(self.monitor) + + @cached_property + def unit(self): + return self.resource.unit + + +def create_resource_relation(): + class ResourceHandler(object): + """ account.resources.web """ + def __getattr__(self, attr): + """ get or build ResourceData """ + if attr.startswith('_'): + raise AttributeError + try: + return self.obj.__resource_cache[attr] + except AttributeError: + self.obj.__resource_cache = {} + except KeyError: + pass + try: + rdata = self.obj.resource_set.get(resource__name=attr) + except ResourceData.DoesNotExist: + model = self.obj._meta.model_name + resource = Resource.objects.get( + content_type__model=model, + name=attr, + is_active=True + ) + rdata = ResourceData( + content_object=self.obj, + content_object_repr=str(self.obj), + resource=resource, + allocated=resource.default_allocation + ) + self.obj.__resource_cache[attr] = rdata + return rdata + + def __get__(self, obj, cls): + """ proxy handled object """ + self.obj = obj + return self + + def __iter__(self): + return iter(self.obj.resource_set.all()) + + # Clean previous state + for related in Resource._related: + try: + delattr(related, 'resource_set') + delattr(related, 'resources') + except AttributeError: + pass + else: + related._meta.private_fields = [ + field for field in related._meta.private_fields if field.remote_field.model != ResourceData + ] + + for ct, resources in Resource.objects.group_by('content_type').items(): + model = ct.model_class() + relation = GenericRelation('resources.ResourceData') + model.add_to_class('resource_set', relation) + model.resources = ResourceHandler() + Resource._related.add(model) diff --git a/orchestra/contrib/resources/serializers.py b/orchestra/contrib/resources/serializers.py new file mode 100644 index 00000000..140674ed --- /dev/null +++ b/orchestra/contrib/resources/serializers.py @@ -0,0 +1,93 @@ +from rest_framework import serializers + +from orchestra.api import router +from orchestra.utils.db import database_ready + +from .models import Resource, ResourceData + + +class ResourceSerializer(serializers.ModelSerializer): + name = serializers.SerializerMethodField() + unit = serializers.ReadOnlyField() + + class Meta: + model = ResourceData + fields = ('name', 'used', 'allocated', 'unit') + read_only_fields = ('used',) + + def to_internal_value(self, raw_data): + data = super(ResourceSerializer, self).to_internal_value(raw_data) + if not data.resource_id: + data.resource = Resource.objects.get(name=raw_data['name']) + return data + + def get_name(self, instance): + return instance.resource.name + + def get_identity(self, data): + return data.get('name') + + +# Monkey-patching section + +def insert_resource_serializers(): + # clean previous state + for related in Resource._related: + try: + viewset = router.get_viewset(related) + except KeyError: + # API viewset not registered + pass + else: + fields = list(viewset.serializer_class.Meta.fields) + try: + fields.remove('resources') + except ValueError: + pass + viewset.serializer_class.Meta.fields = fields + # Create nested serializers on target models + for ct, resources in Resource.objects.group_by('content_type').items(): + model = ct.model_class() + try: + router.insert(model, 'resources', ResourceSerializer, required=False, many=True, source='resource_set') + except KeyError: + continue + # TODO this is a fucking workaround, reimplement this on the proper place + def validate_resources(self, posted, _resources=resources): + """ Creates missing resources """ + result = [] + resources = list(_resources) + for data in posted: + resource = data.resource + if resource not in resources: + msg = "Unknown or duplicated resource '%s'." % resource + raise serializers.ValidationError(msg) + resources.remove(resource) + if not resource.on_demand and not data.allocated: + data.allocated = resource.default_allocation + result.append(data) + for resource in resources: + data = ResourceData(resource=resource) + if not resource.on_demand: + data.allocated = resource.default_allocation + result.append(data) + return result + viewset = router.get_viewset(model) + viewset.serializer_class.validate_resources = validate_resources + + old_options = viewset.options + def options(self, request, resources=resources): + """ Provides available resources description """ + metadata = old_options(self, request) + metadata.data['available_resources'] = [ + { + 'name': resource.name, + 'on_demand': resource.on_demand, + 'default_allocation': resource.default_allocation + } for resource in resources + ] + return metadata + viewset.options = options + +if database_ready(): + insert_resource_serializers() diff --git a/orchestra/contrib/resources/settings.py b/orchestra/contrib/resources/settings.py new file mode 100644 index 00000000..be4ac62d --- /dev/null +++ b/orchestra/contrib/resources/settings.py @@ -0,0 +1,6 @@ +from orchestra.contrib.settings import Setting + + +RESOURCES_OLD_MONITOR_DATA_DAYS = Setting('RESOURCES_OLD_MONITOR_DATA_DAYS', + 40, +) diff --git a/orchestra/contrib/resources/signals.py b/orchestra/contrib/resources/signals.py new file mode 100644 index 00000000..6ce7376a --- /dev/null +++ b/orchestra/contrib/resources/signals.py @@ -0,0 +1,18 @@ +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver + +from .models import Resource + + +@receiver(post_save, sender=Resource, dispatch_uid="resources.sync_periodic_task") +def sync_periodic_task(sender, **kwargs): + """ useing signals instead of Model.delete() override beucause of admin bulk delete() """ + instance = kwargs['instance'] + instance.sync_periodic_task() + + +@receiver(post_delete, sender=Resource, dispatch_uid="resources.delete_periodic_task") +def delete_periodic_task(sender, **kwargs): + """ useing signals instead of Model.delete() override beucause of admin bulk delete() """ + instance = kwargs['instance'] + instance.sync_periodic_task(delete=True) diff --git a/orchestra/contrib/resources/tasks.py b/orchestra/contrib/resources/tasks.py new file mode 100644 index 00000000..10f80729 --- /dev/null +++ b/orchestra/contrib/resources/tasks.py @@ -0,0 +1,77 @@ +import datetime + +from celery.task.schedules import crontab +from django.db import transaction +from django.utils import timezone + +from orchestra.contrib.orchestration import Operation +from orchestra.contrib.tasks import task, periodic_task +from orchestra.models.utils import get_model_field_path +from orchestra.utils.sys import LockFile + +from . import settings +from .backends import ServiceMonitor + + +@task(name='resources.Monitor') +def monitor(resource_id, ids=None): + with LockFile('/dev/shm/resources.monitor-%i.lock' % resource_id, expire=60*60, unlocked=bool(ids)): + from .models import ResourceData, Resource + resource = Resource.objects.get(pk=resource_id) + resource_model = resource.content_type.model_class() + logs = [] + # Execute monitors + for monitor_name in resource.monitors: + backend = ServiceMonitor.get_backend(monitor_name) + model = backend.model_class() + kwargs = {} + if ids: + path = get_model_field_path(model, resource_model) + path = '%s__in' % ('__'.join(path) or 'id') + kwargs = { + path: ids + } + # Execute monitor + monitorings = [] + for obj in model.objects.filter(**kwargs): + op = Operation(backend, obj, Operation.MONITOR) + monitorings.append(op) + logs += Operation.execute(monitorings, run_async=False) + + kwargs = {'id__in': ids} if ids else {} + # Update used resources and trigger resource exceeded and revovery + triggers = [] + model = resource.content_type.model_class() + for obj in model.objects.filter(**kwargs): + data, __ = ResourceData.objects.get_or_create(obj, resource) + data.update() + if not resource.disable_trigger: + a = data.used + b = data.allocated + if data.used > (data.allocated or 0): + op = Operation(backend, obj, Operation.EXCEEDED) + triggers.append(op) + elif data.used < (data.allocated or 0): + op = Operation(backend, obj, Operation.RECOVERY) + triggers.append(op) + Operation.execute(triggers) + return logs + + +@periodic_task(run_every=crontab(hour=2, minute=30), name='resources.cleanup_old_monitors') +@transaction.atomic +def cleanup_old_monitors(queryset=None): + if queryset is None: + from .models import MonitorData + queryset = MonitorData.objects.all() + delta = datetime.timedelta(days=settings.RESOURCES_OLD_MONITOR_DATA_DAYS) + threshold = timezone.now() - delta + queryset = queryset.filter(created_at__lt=threshold) + delete_counts = [] + for monitor in ServiceMonitor.get_plugins(): + dataset = queryset.filter(monitor=monitor) + delete_count = monitor.aggregate(dataset) + delete_counts.append( + (monitor.get_name(), delete_count) + ) + return delete_counts diff --git a/orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html b/orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html new file mode 100644 index 00000000..e98eb69c --- /dev/null +++ b/orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html @@ -0,0 +1,251 @@ +{% load i18n utils static %} + + + + Resource history + + + + + + + + + +
    ♦Notice that resources used by deleted services will not appear.
    +
    +
    + > crunching data ... +
    +
    + + diff --git a/orchestra/contrib/resources/validators.py b/orchestra/contrib/resources/validators.py new file mode 100644 index 00000000..710fb5f5 --- /dev/null +++ b/orchestra/contrib/resources/validators.py @@ -0,0 +1,11 @@ +from django.core.validators import ValidationError +from django.utils.translation import gettext_lazy as _ + + +def validate_scale(value): + try: + int(eval(value)) + except Exception as e: + raise ValidationError( + _("'%s' is not a valid scale expression. (%s)") % (value, e) + ) diff --git a/orchestra/contrib/saas/README.md b/orchestra/contrib/saas/README.md new file mode 100644 index 00000000..46fcb88f --- /dev/null +++ b/orchestra/contrib/saas/README.md @@ -0,0 +1,90 @@ + + +# SaaS - Software as a Service + + +This app provides support for services that follow the SaaS model. Traditionally known as multi-site or multi-tenant web applications where a single installation of a CMS provides accounts for multiple isolated tenants. + + +## Service declaration + +Each service is defined by a `SoftwareService` subclass, you can find examples on the [`services` module](services). + +The minimal service declaration will be: + +```python +class DrupalService(SoftwareService): + name = 'drupal' + verbose_name = "Drupal" + icon = 'orchestra/icons/apps/Drupal.png' + site_domain = settings.SAAS_MOODLE_DOMAIN +``` + +Additional attributes can be used to further customize the service class to your needs. + +### Custom forms +If a service needs to keep track of additional information (other than a user/site name, is_active, custom_url, or database) an extra form and serializer should be provided. For example, WordPress requires to provide an *email address* for account creation, and the assigned *blog ID* is required for effectively identify the account for update and delete operations. In this case we provide two forms, one for account creation and another for change: + +```python +class WordPressForm(SaaSBaseForm): + email = forms.EmailField(label=_("Email"), + help_text=_("A new user will be created if the above email address is not in the database.
    " + "The username and password will be mailed to this email address.")) + +class WordPressChangeForm(WordPressForm): + blog_id = forms.IntegerField(label=("Blog ID"), widget=widgets.SpanWidget, required=False, + help_text=_("ID of this blog used by WordPress, the only attribute that doesn't change.")) +``` + +`WordPressForm` provides the email field, and `WordPressChangeForm` adds the `blog_id` on top of it. `blog_id` will be represented as a *readonly* field on the form (`widget=widgets.SpanWidget`), so no modification will be allowed. + +Additionally, `SaaSPasswordForm` provides a password field for the common case when a password needs to be provided in order to create a new account. You can subclass `SaaSPasswordForm` or use it directly on the `Service.form` field. + + +### Serializer for extra data + +In case we need to save extra information of the service (email and blog_id in our current example) we should provide a serializer that serializes this bits of information into JSON format so they can be saved and retrieved from the database data field. + +```python +class WordPressDataSerializer(serializers.Serializer): + email = serializers.EmailField(label=_("Email")) + blog_id = serializers.IntegerField(label=_("Blog ID"), allow_null=True, required=False) +``` + +Now we have everything needed for declaring the WordPress service. + +```python +class WordPressService(SoftwareService): + name = 'wordpress' + verbose_name = "WordPress" + form = WordPressForm + change_form = WordPressChangeForm + serializer = WordPressDataSerializer + icon = 'orchestra/icons/apps/WordPress.png' + change_readonly_fields = ('email', 'blog_id') + site_domain = settings.SAAS_WORDPRESS_DOMAIN + allow_custom_url = settings.SAAS_WORDPRESS_ALLOW_CUSTOM_URL +``` + +Notice that two optional forms can be provided `form` and `change_form`. When non of them is provided, SaaS will provide a default one for you. When only `form` is provided, it will be used for both, *add view* and *change view*. If both are provided, `form` will be used for the *add view* and `change_form` for the *change view*. This last option allows us to display the `blog_id` back to the user, only when we know its value (after creation). + +`change_readonly_fields` is a tuple with the name of the fields that can **not** be edited once the service has been created. + +`allow_custom_url` is a boolean flag that defines whether this service is allowed to have custom URL's (URL of any form) or not. In case it does, additional steps are required for interfacing with `orchestra.contrib.websites`, such as having an enabled website directive (`WEBSITES_ENABLED_DIRECTIVES`) that knows where the SaaS webapp is running, such as `'orchestra.contrib.websites.directives.WordPressSaaS'`. + + +## Backend +A backend class is required to interface with the web application and perform `save()` and `delete()` operations on it. + +- The more reliable way of interfacing with the application is by means of a CLI (e.g. [Moodle](backends/moodle.py)), but not all CMS come with this tool. +- The second preferable way is using some sort of networked API, possibly HTTP-based (e.g. [gitLab](backends/gitlab.py)). This is less reliable because additional moving parts are used underneath the interface; a busy web server can timeout our requests. +- The least preferred way is interfacing with an HTTP-HTML interface designed for human consumption, really painful to implement but sometimes is the only way (e.g. [WordPress](backends/wordpressmu.py)). + +Some applications do not support multi-tenancy by default, but we can hack the configuration file of such apps and generate *table prefix* or *database name* based on some property of the URL. Example of this services are [moodle](backends/moodle.py) and [phplist](backends/phplist.py) respectively. + + +## Settings + +Enabled services should be added into the `SAAS_ENABLED_SERVICES` settings tuple, providing its full module path, e.g. `'orchestra.contrib.saas.services.moodle.MoodleService'`. + +Parameters that should allow easy configuration on each deployment should be defined as settings. e.g. `SAAS_WORDPRESS_DOMAIN`. Take a look at the [`settings` module](settings.py). diff --git a/orchestra/contrib/saas/__init__.py b/orchestra/contrib/saas/__init__.py new file mode 100644 index 00000000..0bc8af8b --- /dev/null +++ b/orchestra/contrib/saas/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.saas.apps.SaaSConfig' diff --git a/orchestra/contrib/saas/admin.py b/orchestra/contrib/saas/admin.py new file mode 100644 index 00000000..90176d87 --- /dev/null +++ b/orchestra/contrib/saas/admin.py @@ -0,0 +1,60 @@ +from django.contrib import admin +from django.core.exceptions import ObjectDoesNotExist +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin +from orchestra.admin.actions import disable, enable +from orchestra.admin.utils import change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter +from orchestra.plugins.admin import SelectPluginAdminMixin +from orchestra.utils.apps import isinstalled +from orchestra.utils.html import get_on_site_link + +from .filters import CustomURLListFilter +from .models import SaaS +from .services import SoftwareService + + +class SaaSAdmin(SelectPluginAdminMixin, ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdmin): + list_display = ('name', 'service', 'display_url', 'account_link', 'display_active') + list_filter = ('service', IsActiveListFilter, CustomURLListFilter) + search_fields = ('name', 'account__username') + change_readonly_fields = ('service',) + plugin = SoftwareService + plugin_field = 'service' + plugin_title = 'Software as a Service' + actions = (disable, enable, list_accounts) + + @mark_safe + def display_url(self, saas): + site_domain = saas.get_site_domain() + site_link = '%s' % (site_domain, site_domain) + links = [site_link] + if saas.custom_url and isinstalled('orchestra.contrib.websites'): + try: + website = saas.service_instance.get_website() + except ObjectDoesNotExist: + warning = _("Related website directive does not exist for this custom URL.") + link = '%s' % (warning, saas.custom_url) + else: + website_link = get_on_site_link(saas.custom_url) + admin_url = change_url(website) + link = '%s %s' % ( + admin_url, saas.custom_url, website_link + ) + links.append(link) + return '
    '.join(links) + display_url.short_description = _("URL") + display_url.admin_order_field = 'name' + + def get_fields(self, *args, **kwargs): + fields = super(SaaSAdmin, self).get_fields(*args, **kwargs) + if not self.plugin_instance.allow_custom_url: + return [field for field in fields if field != 'custom_url'] + return fields + + +admin.site.register(SaaS, SaaSAdmin) diff --git a/orchestra/contrib/saas/api.py b/orchestra/contrib/saas/api.py new file mode 100644 index 00000000..de226b38 --- /dev/null +++ b/orchestra/contrib/saas/api.py @@ -0,0 +1,16 @@ +from rest_framework import viewsets + +from orchestra.api import router, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import SaaS +from .serializers import SaaSSerializer + + +class SaaSViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = SaaS.objects.all() + serializer_class = SaaSSerializer + filter_fields = ('name',) + + +router.register(r'saas', SaaSViewSet) diff --git a/orchestra/contrib/saas/apps.py b/orchestra/contrib/saas/apps.py new file mode 100644 index 00000000..8ad3f8c8 --- /dev/null +++ b/orchestra/contrib/saas/apps.py @@ -0,0 +1,13 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class SaaSConfig(AppConfig): + name = 'orchestra.contrib.saas' + verbose_name = 'Saas' + + def ready(self): + from . import signals + from .models import SaaS + services.register(SaaS, icon='saas.png') diff --git a/orchestra/contrib/saas/backends/__init__.py b/orchestra/contrib/saas/backends/__init__.py new file mode 100644 index 00000000..b1c07976 --- /dev/null +++ b/orchestra/contrib/saas/backends/__init__.py @@ -0,0 +1,132 @@ +import pkgutil +import textwrap + +from orchestra.contrib.resources import ServiceMonitor + +from .. import settings + + +class ApacheTrafficByHost(ServiceMonitor): + """ + Parses apache logs, + looking for the size of each request on the last word of the log line. + + Compatible log format: + LogFormat "%h %l %u %t \"%r\" %>s %O %{Host}i" host + or if include_received_bytes: + LogFormat "%h %l %u %t \"%r\" %>s %I %O %{Host}i" host + CustomLog /home/pangea/logs/apache/host_blog.pangea.org.log host + """ + model = 'saas.SaaS' + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + abstract = True + include_received_bytes = False + + def prepare(self): + access_log = self.log_path + context = { + 'access_logs': str((access_log, access_log+'.1')), + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS), + 'include_received_bytes': str(self.include_received_bytes), + } + self.append(textwrap.dedent("""\ + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + access_logs = {access_logs} + sites = {{}} + months = {{ + 'Jan': '01', + 'Feb': '02', + 'Mar': '03', + 'Apr': '04', + 'May': '05', + 'Jun': '06', + 'Jul': '07', + 'Aug': '08', + 'Sep': '09', + 'Oct': '10', + 'Nov': '11', + 'Dec': '12', + }} + + def prepare(object_id, site_domain, ini_date): + global sites + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + sites[site_domain] = [ini_date, object_id, 0] + + def monitor(sites, end_date, months, access_logs): + include_received = {include_received_bytes} + for access_log in access_logs: + try: + with open(access_log, 'r') as handler: + for line in handler.readlines(): + line = line.split() + host, __, __, date = line[:4] + if host in {ignore_hosts}: + continue + size, hostname = line[-2:] + size = int(size) + if include_received: + size += int(line[-3]) + try: + site = sites[hostname] + except KeyError: + continue + else: + # [16/Sep/2015:11:40:38 + day, month, date = date[1:].split('/') + year, hour, min, sec = date.split(':') + date = year + months[month] + day + hour + min + sec + if site[0] < int(date) < end_date: + site[2] += size + except IOError as e: + sys.stderr.write(str(e)+'\\n') + for opts in sites.values(): + ini_date, object_id, size = opts + sys.stdout.write('%s %s\\n' % (object_id, size)) + """).format(**context) + ) + + def monitor(self, saas): + context = self.get_context(saas) + self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context) + + def commit(self): + self.append('monitor(sites, end_date, months, access_logs)') + + def get_context(self, saas): + return { + 'site_domain': saas.get_site_domain(), + 'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'object_id': saas.pk, + } + + +class ApacheTrafficByName(ApacheTrafficByHost): + __doc__ = ApacheTrafficByHost.__doc__ + + def get_context(self, saas): + return { + 'site_domain': saas.name, + 'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'object_id': saas.pk, + } + + +for __, module_name, __ in pkgutil.walk_packages(__path__): + # sorry for the exec(), but Import module function fails :( + exec('from . import %s' % module_name) diff --git a/orchestra/contrib/saas/backends/bscw.py b/orchestra/contrib/saas/backends/bscw.py new file mode 100644 index 00000000..0d0a1156 --- /dev/null +++ b/orchestra/contrib/saas/backends/bscw.py @@ -0,0 +1,59 @@ +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace + +from .. import settings + + +class BSCWController(ServiceController): + verbose_name = _("BSCW SaaS") + model = 'saas.SaaS' + default_route_match = "saas.service == 'bscw'" + actions = ('save', 'delete', 'validate_creation') + doc_settings = (settings, + ('SAAS_BSCW_BSADMIN_PATH',) + ) + + def validate_creation(self, saas): + context = self.get_context(saas) + self.append(textwrap.dedent("""\ + if [[ $(%(bsadmin)s register %(email)s) ]]; then + echo 'ValidationError: email-exists' + fi + if [[ $(%(bsadmin)s users -n %(username)s) ]]; then + echo 'ValidationError: user-exists' + fi""") % context + ) + + def save(self, saas): + context = self.get_context(saas) + if hasattr(saas, 'password'): + self.append(textwrap.dedent("""\ + if [[ ! $(%(bsadmin)s register %(email)s) && ! $(%(bsadmin)s users -n %(username)s) ]]; then + # Create new user + %(bsadmin)s register -r %(email)s %(username)s '%(password)s' + else + # Change password + %(bsadmin)s chpwd %(username)s '%(password)s' + fi + """) % context + ) + elif saas.active: + self.append("%(bsadmin)s chpwd -u %(username)s" % context) + else: + self.append("%(bsadmin)s chpwd -l %(username)s" % context) + + def delete(self, saas): + context = self.get_context(saas) + self.append("%(bsadmin)s rmuser -n %(username)s" % context) + + def get_context(self, saas): + context = { + 'bsadmin': settings.SAAS_BSCW_BSADMIN_PATH, + 'email': saas.data.get('email'), + 'username': saas.name, + 'password': getattr(saas, 'password', None), + } + return replace(context, "'", '"') diff --git a/orchestra/contrib/saas/backends/dokuwikimu.py b/orchestra/contrib/saas/backends/dokuwikimu.py new file mode 100644 index 00000000..21a4d44e --- /dev/null +++ b/orchestra/contrib/saas/backends/dokuwikimu.py @@ -0,0 +1,115 @@ +import crypt +import os +import textwrap +from urllib.parse import urlparse + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController +from orchestra.utils.python import random_ascii + +from . import ApacheTrafficByHost +from .. import settings + + +class DokuWikiMuController(ServiceController): + """ + Creates a DokuWiki site on a DokuWiki multisite installation. + """ + name = 'dokuwiki' + verbose_name = _("DokuWiki multisite") + model = 'saas.SaaS' + default_route_match = "saas.service == 'dokuwiki'" + doc_settings = (settings, ( + 'SAAS_DOKUWIKI_TEMPLATE_PATH', + 'SAAS_DOKUWIKI_FARM_PATH', + 'SAAS_DOKUWIKI_USER', + 'SAAS_DOKUWIKI_GROUP', + )) + + def save(self, saas): + context = self.get_context(saas) + self.append(textwrap.dedent(""" + if [[ ! -e %(app_path)s ]]; then + mkdir %(app_path)s + tar xfz %(template)s -C %(app_path)s + chown -R %(user)s:%(group)s %(app_path)s + fi""") % context + ) + if context['password']: + self.append(textwrap.dedent("""\ + if grep '^admin:' %(users_path)s > /dev/null; then + sed -i 's#^admin:.*$#admin:%(password)s:admin:%(email)s:admin,user#' %(users_path)s + else + echo 'admin:%(password)s:admin:%(email)s:admin,user' >> %(users_path)s + fi""") % context + ) + self.append(textwrap.dedent("""\ + # Update custom domain link + find %(farm_path)s \\ + -maxdepth 1 \\ + -type l \\ + -exec bash -c ' + if [[ $(readlink {}) == "%(domain)s" && $(basename {}) != "%(custom_domain)s" ]]; then + rm {} + fi' \;\ + """) % context + ) + if context['custom_domain']: + self.append(textwrap.dedent("""\ + if [[ ! -e %(farm_path)s/%(custom_domain)s ]]; then + ln -s %(domain)s %(farm_path)s/%(custom_domain)s + chown -h %(user)s:%(group) %(farm_path)s/%(custom_domain)s + fi""") % context + ) + + def delete(self, saas): + context = self.get_context(saas) + self.append("rm -fr %(app_path)s" % context) + self.append(textwrap.dedent("""\ + # Delete custom domain link + find %(farm_path)s \\ + -maxdepth 1 \\ + -type l \\ + -exec bash -c ' + if [[ $(readlink {}) == "%(domain)s" ]]; then + rm {} + fi' \;\ + """) % context + ) + + def get_context(self, saas): + context = super(DokuWikiMuController, self).get_context(saas) + domain = saas.get_site_domain() + context.update({ + 'template': settings.SAAS_DOKUWIKI_TEMPLATE_PATH, + 'farm_path': os.path.normpath(settings.SAAS_DOKUWIKI_FARM_PATH), + 'app_path': os.path.join(settings.SAAS_DOKUWIKI_FARM_PATH, domain), + 'user': settings.SAAS_DOKUWIKI_USER, + 'group': settings.SAAS_DOKUWIKI_GROUP, + 'email': saas.account.email, + 'custom_url': saas.custom_url, + 'domain': domain, + }) + if saas.custom_url: + custom_url = urlparse(saas.custom_url) + context.update({ + 'custom_domain': custom_url.netloc, + }) + password = getattr(saas, 'password', None) + salt = random_ascii(8) + context.update({ + 'password': crypt.crypt(password, '$1$'+salt) if password else None, + 'users_path': os.path.join(context['app_path'], 'conf/users.auth.php'), + }) + return context + + +class DokuWikiMuTraffic(ApacheTrafficByHost): + __doc__ = ApacheTrafficByHost.__doc__ + verbose_name = _("DokuWiki MU Traffic") + default_route_match = "saas.service == 'dokuwiki'" + doc_settings = (settings, + ('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_DOKUWIKI_LOG_PATH') + ) + log_path = settings.SAAS_DOKUWIKI_LOG_PATH diff --git a/orchestra/contrib/saas/backends/drupalmu.py b/orchestra/contrib/saas/backends/drupalmu.py new file mode 100644 index 00000000..944903c9 --- /dev/null +++ b/orchestra/contrib/saas/backends/drupalmu.py @@ -0,0 +1,46 @@ +import os +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace + +from .. import settings + + +class DrupalMuController(ServiceController): + """ + Creates a Drupal site on a Drupal multisite installation + """ + verbose_name = _("Drupal multisite") + model = 'saas.SaaS' + default_route_match = "saas.service == 'drupal'" + doc_settings = (settings, + ('SAAS_DRUPAL_SITES_PATH',) + ) + + def save(self, webapp): + context = self.get_context(webapp) + # TODO set password + self.append(textwrap.dedent("""\ + mkdir %(drupal_path)s + chown -R www-data %(drupal_path)s + + # the following assumes settings.php to be previously configured + REGEX='^\s*$databases\[.default.\]\[.default.\]\[.prefix.\]' + CONFIG='$databases[\'default\'][\'default\'][\'prefix\'] = \'%(app_name)s_\';' + if ! grep $REGEX %(drupal_settings)s > /dev/null; then + echo $CONFIG >> %(drupal_settings)s + fi""") % context + ) + + def delete(self, webapp): + context = self.get_context(webapp) + # TODO delete tables + self.append("rm -fr %(app_path)s" % context) + + def get_context(self, webapp): + context = super(DrupalMuController, self).get_context(webapp) + context['drupal_path'] = settings.SAAS_DRUPAL_SITES_PATH % context + context['drupal_settings'] = os.path.join(context['drupal_path'], 'settings.php') + return replace(context, "'", '"') diff --git a/orchestra/contrib/saas/backends/gitlab.py b/orchestra/contrib/saas/backends/gitlab.py new file mode 100644 index 00000000..042c82ae --- /dev/null +++ b/orchestra/contrib/saas/backends/gitlab.py @@ -0,0 +1,119 @@ +import json + +import requests +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController + +from .. import settings + + +class GitLabSaaSController(ServiceController): + verbose_name = _("GitLab SaaS") + model = 'saas.SaaS' + default_route_match = "saas.service == 'gitlab'" + serialize = True + actions = ('save', 'delete', 'validate_creation') + doc_settings = (settings, + ('SAAS_GITLAB_DOMAIN', 'SAAS_GITLAB_ROOT_PASSWORD', 'SAAS_GITLAB_VERIFY_SSL'), + ) + verify = settings.SAAS_GITLAB_VERIFY_SSL + + def get_base_url(self): + return 'https://%s/api/v3' % settings.SAAS_GITLAB_DOMAIN + + def get_user_url(self, saas): + user_id = saas.data['user_id'] + return self.get_base_url() + '/users/%i' % user_id + + def validate_response(self, response, *status_codes): + if response.status_code not in status_codes: + raise RuntimeError("[%i] %s" % (response.status_code, response.content)) + return response.json() + + def authenticate(self): + login_url = self.get_base_url() + '/session' + data = { + 'login': 'root', + 'password': settings.SAAS_GITLAB_ROOT_PASSWORD, + } + response = requests.post(login_url, data=data, verify=self.verify) + session = self.validate_response(response, 201) + token = session['private_token'] + self.headers = { + 'PRIVATE-TOKEN': token, + } + + def create_user(self, saas, server): + self.authenticate() + user_url = self.get_base_url() + '/users' + data = { + 'email': saas.data['email'], + 'password': saas.password, + 'username': saas.name, + 'name': saas.account.get_full_name(), + } + response = requests.post(user_url, data=data, headers=self.headers, verify=self.verify) + user = self.validate_response(response, 201) + saas.data['user_id'] = user['id'] + # Using queryset update to avoid triggering backends with the post_save signal + type(saas).objects.filter(pk=saas.pk).update(data=saas.data) + print(json.dumps(user, indent=4)) + + def change_password(self, saas, server): + self.authenticate() + user_url = self.get_user_url(saas) + response = requests.get(user_url, headers=self.headers, verify=self.verify) + user = self.validate_response(response, 200) + user = response.json() + user['password'] = saas.password + response = requests.put(user_url, data=user, headers=self.headers, verify=self.verify) + user = self.validate_response(response, 200) + print(json.dumps(user, indent=4)) + + def set_state(self, saas, server): + # TODO http://feedback.gitlab.com/forums/176466-general/suggestions/4098632-add-administrative-api-call-to-block-users + return + self.authenticate() + user_url = self.get_user_url(saas) + response = requests.get(user_url, headers=self.headers, verify=self.verify) + user = self.validate_response(response, 200) + user['state'] = 'active' if saas.active else 'blocked', + response = requests.patch(user_url, data=user, headers=self.headers, verify=self.verify) + user = self.validate_response(response, 200) + print(json.dumps(user, indent=4)) + + def delete_user(self, saas, server): + self.authenticate() + user_url = self.get_user_url(saas) + response = requests.delete(user_url, headers=self.headers, verify=self.verify) + user = self.validate_response(response, 200, 404) + print(json.dumps(user, indent=4)) + + def _validate_creation(self, saas, server): + """ checks if a saas object is valid for creation on the server side """ + self.authenticate() + username = saas.name + email = saas.data['email'] + users_url = self.get_base_url() + '/users/' + response = requests.get(users_url, headers=self.headers, verify=self.verify) + users = response.json() + for user in users: + if user['username'] == username: + print('ValidationError: user-exists') + if user['email'] == email: + print('ValidationError: email-exists') + + def validate_creation(self, saas): + self.append(self._validate_creation, saas) + + def save(self, saas): + if hasattr(saas, 'password'): + if saas.data.get('user_id', None): + self.append(self.change_password, saas) + else: + self.append(self.create_user, saas) + self.append(self.set_state, saas) + + def delete(self, saas): + self.append(self.delete_user, saas) diff --git a/orchestra/contrib/saas/backends/moodle.py b/orchestra/contrib/saas/backends/moodle.py new file mode 100644 index 00000000..d942675e --- /dev/null +++ b/orchestra/contrib/saas/backends/moodle.py @@ -0,0 +1,170 @@ +import textwrap +from urllib.parse import urlparse + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController + +from .. import settings + + +class MoodleMuController(ServiceController): + """ + Creates a Moodle site on a Moodle multisite installation + + // config.php + // map custom domains to sites + $site_map = array( + // "" => ["", ""], + ); + + $site = getenv("SITE"); + if ( $site == '' ) { + $http_host = $_SERVER['HTTP_HOST']; + if (array_key_exists($http_host, $site_map)) { + $site = $site_map[$http_host][0]; + $wwwroot = $site_map[$http_host][1]; + } elseif (strpos($http_host, '-courses.') !== false) { + $site = array_shift((explode("-courses.", $http_host))); + $wwwroot = "https://{$site}-courses.pangea.org"; + } else { + $site = array_shift((explode(".", $http_host))); + $wwwroot = "https://{$site}-courses.pangea.org"; + } + } else { + $wwwroot = "https://{$site}-courses.pangea.org"; + foreach ($site_map as $key => $value) { + if ($value[0] == $site) { + $wwwroot = $value[1]; + break; + } + } + } + + $prefix = str_replace('-', '_', $site); + $CFG->prefix = "${prefix}_"; + $CFG->wwwroot = $wwwroot; + $CFG->dataroot = "/home/pangea/moodledata/{$site}/"; + """ + verbose_name = _("Moodle multisite") + model = 'saas.SaaS' + default_route_match = "saas.service == 'moodle'" + + def save(self, webapp): + context = self.get_context(webapp) + self.delete_site_map(context) + if context['custom_url']: + self.insert_site_map(context) + self.append(textwrap.dedent("""\ + mkdir -p %(moodledata_path)s + chown %(user)s:%(user)s %(moodledata_path)s + export SITE=%(site_name)s + CHANGE_PASSWORD=0 + # TODO su moodle user + php %(moodle_path)s/admin/cli/install_database.php \\ + --fullname="%(site_name)s" \\ + --shortname="%(site_name)s" \\ + --adminpass="%(password)s" \\ + --adminemail="%(email)s" \\ + --non-interactive \\ + --agree-license \\ + --allow-unstable || CHANGE_PASSWORD=1 + """) % context + ) + if context['password']: + self.append(textwrap.dedent("""\ + mysql \\ + --host="%(db_host)s" \\ + --user="%(db_user)s" \\ + --password="%(db_pass)s" \\ + --execute='UPDATE %(db_prefix)s_user + SET password=MD5("%(password)s") + WHERE username="admin";' \\ + %(db_name)s + """) % context + ) + if context['crontab']: + context['escaped_crontab'] = context['crontab'].replace('$', '\\$') + self.append(textwrap.dedent("""\ + # Configuring Moodle crontabs + if ! crontab -u %(user)s -l | grep 'Moodle:"%(site_name)s"' > /dev/null; then + cat << EOF | su - %(user)s --shell /bin/bash -c 'crontab' + $(crontab -u %(user)s -l) + + # %(banner)s - Moodle:"%(site_name)s" + %(escaped_crontab)s + EOF + fi""") % context + ) + + def delete_site_map(self, context): + self.append(textwrap.dedent("""\ + sed -i '/^\s*"[^\s]*"\s*=>\s*\["%(site_name)s",\s*".*/d' %(moodle_path)s/config.php + """) % context + ) + + def insert_site_map(self, context): + self.append(textwrap.dedent("""\ + regex='\s*\$site_map\s+=\s+array\(' + newline=' "%(custom_domain)s" => ["%(site_name)s", "%(custom_url)s"], // %(banner)s' + sed -i -r "s#$regex#\$site_map = array(\\n$newline#" %(moodle_path)s/config.php + """) % context + ) + + def delete(self, saas): + context = self.get_context(saas) + self.append(textwrap.dedent(""" + rm -rf %(moodledata_path)s + # Delete tables with prefix %(db_prefix)s + mysql -Nrs \\ + --host="%(db_host)s" \\ + --user="%(db_user)s" \\ + --password="%(db_pass)s" \\ + --execute='SET GROUP_CONCAT_MAX_LEN=10000; + SET @tbls = (SELECT GROUP_CONCAT(TABLE_NAME) + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = "%(db_name)s" + AND TABLE_NAME LIKE "%(db_prefix)s_%%"); + SET @delStmt = CONCAT("DROP TABLE ", @tbls); + -- SELECT @delStmt; + PREPARE stmt FROM @delStmt; + EXECUTE stmt; + DEALLOCATE PREPARE stmt;' \\ + %(db_name)s + """) % context + ) + if context['crontab']: + context['crontab_regex'] = '\\|'.join(context['crontab'].splitlines()) + context['crontab_regex'] = context['crontab_regex'].replace('*', '\\*') + self.append(textwrap.dedent("""\ + crontab -u %(user)s -l \\ + | grep -v 'Moodle:"%(site_name)s"\\|%(crontab_regex)s' \\ + | su - %(user)s --shell /bin/bash -c 'crontab' + """) % context + ) + self.delete_site_map(context) + + def get_context(self, saas): + context = { + 'banner': self.get_banner(), + 'name': saas.name, + 'site_name': saas.name, + 'full_name': "%s course" % saas.name.capitalize(), + 'moodle_path': settings.SAAS_MOODLE_PATH, + 'user': settings.SAAS_MOODLE_SYSTEMUSER, + 'db_user': settings.SAAS_MOODLE_DB_USER, + 'db_pass': settings.SAAS_MOODLE_DB_PASS, + 'db_name': settings.SAAS_MOODLE_DB_NAME, + 'db_host': settings.SAAS_MOODLE_DB_HOST, + 'db_prefix': saas.name.replace('-', '_'), + 'email': saas.account.email, + 'password': getattr(saas, 'password', None), + 'custom_url': saas.custom_url.rstrip('/'), + 'custom_domain': urlparse(saas.custom_url).netloc if saas.custom_url else None, + } + context.update({ + 'crontab': settings.SAAS_MOODLE_CRONTAB % context, + 'db_name': context['db_name'] % context, + 'moodledata_path': settings.SAAS_MOODLE_DATA_PATH % context, + }) + return context diff --git a/orchestra/contrib/saas/backends/nextcloud.py b/orchestra/contrib/saas/backends/nextcloud.py new file mode 100644 index 00000000..17df098f --- /dev/null +++ b/orchestra/contrib/saas/backends/nextcloud.py @@ -0,0 +1,175 @@ +import re +import sys +import textwrap +import time +import xml.etree.ElementTree as ET +from urllib.parse import urlparse + +import requests +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor + +from . import ApacheTrafficByName +from .. import settings + + +class NextCloudAPIMixin(object): + def validate_response(self, response): + request = response.request + context = (request.method, response.url, request.body, response.status_code) + sys.stderr.write("%s %s '%s' HTTP %s\n" % context) + if response.status_code != requests.codes.ok: + raise RuntimeError("%s %s '%s' HTTP %s" % context) + root = ET.fromstring(response.text) + statuscode = root.find("./meta/statuscode").text + if statuscode != '100': + message = root.find("./meta/status").text + request = response.request + context = (request.method, response.url, request.body, statuscode, message) + raise RuntimeError("%s %s '%s' ERROR %s, %s" % context) + + def api_call(self, action, url_path, *args, **kwargs): + BASE_URL = settings.SAAS_NEXTCLOUD_API_URL.rstrip('/') + url = '/'.join((BASE_URL, url_path)) + response = action(url, headers={'OCS-APIRequest':'true'}, verify=False, *args, **kwargs) + self.validate_response(response) + return response + + def api_get(self, url_path, *args, **kwargs): + return self.api_call(requests.get, url_path, *args, **kwargs) + + def api_post(self, url_path, *args, **kwargs): + return self.api_call(requests.post, url_path, *args, **kwargs) + + def api_put(self, url_path, *args, **kwargs): + return self.api_call(requests.put, url_path, *args, **kwargs) + + def api_delete(self, url_path, *args, **kwargs): + return self.api_call(requests.delete, url_path, *args, **kwargs) + + def create(self, saas): + data = { + 'userid': saas.name, + 'password': saas.password + } + self.api_post('users', data) + + def update(self, saas): + """ + key: email|quota|display|password + value: el valor a modificar. + Si es un email, tornarà un error si la direcció no te la "@" + Si es una quota, sembla que algo per l'estil "5G", "100M", etc. funciona. Quota 0 = infinit + "display" es el display name, no crec que el fem servir, és cosmetic + """ + data = { + 'key': 'password', + 'value': saas.password, + } + self.api_put('users/%s' % saas.name, data) + + def get_user(self, saas): + """ + { + 'displayname' + 'email' + 'quota' => + { + 'free' (en Bytes) + 'relative' (en tant per cent sense signe %, e.g. 68.17) + 'total' (en Bytes) + 'used' (en Bytes) + } + } + """ + response = self.api_get('users/%s' % saas.name) + root = ET.fromstring(response.text) + ret = {} + for data in root.find('./data'): + ret[data.tag] = data.text + ret['quota'] = {} + for data in root.find('.data/quota'): + ret['quota'][data.tag] = data.text + return ret + + +class NextCloudController(NextCloudAPIMixin, ServiceController): + """ + Creates a wordpress site on a WordPress MultiSite installation. + + You should point it to the database server + """ + verbose_name = _("nextCloud SaaS") + model = 'saas.SaaS' + default_route_match = "saas.service == 'nextcloud'" + doc_settings = (settings, + ('SAAS_NEXTCLOUD_API_URL',) + ) + + def update_or_create(self, saas, server): + try: + self.api_get('users/%s' % saas.name) + except RuntimeError: + if getattr(saas, 'password'): + self.create(saas) + else: + raise + else: + if getattr(saas, 'password'): + self.update(saas) + + def remove(self, saas, server): + self.api_delete('users/%s' % saas.name) + + def save(self, saas): + # TODO disable user https://github.com/owncloud/core/issues/12601 + self.append(self.update_or_create, saas) + + def delete(self, saas): + self.append(self.remove, saas) + + +class NextcloudTraffic(ApacheTrafficByName): + __doc__ = ApacheTrafficByName.__doc__ + verbose_name = _("nextCloud SaaS Traffic") + default_route_match = "saas.service == 'nextcloud'" + doc_settings = (settings, + ('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_NEXTCLOUD_LOG_PATH') + ) + log_path = settings.SAAS_NEXTCLOUD_LOG_PATH + + +class NextCloudDiskQuota(NextCloudAPIMixin, ServiceMonitor): + model = 'saas.SaaS' + verbose_name = _("nextCloud SaaS Disk Quota") + default_route_match = "saas.service == 'nextcloud'" + resource = ServiceMonitor.DISK + delete_old_equal_values = True + + def monitor(self, user): + context = self.get_context(user) + self.append("echo %(object_id)s $(monitor %(base_home)s)" % context) + + def get_context(self, user): + context = { + 'object_id': user.pk, + 'base_home': user.get_base_home(), + } + return replace(context, "'", '"') + + def get_quota(self, saas, server): + try: + user = self.get_user(saas) + except requests.exceptions.ConnectionError: + time.sleep(2) + user = self.get_user(saas) + context = { + 'object_id': saas.pk, + 'used': int(user['quota'].get('used', 0)), + } + sys.stdout.write('%(object_id)i %(used)i\n' % context) + + def monitor(self, saas): + self.append(self.get_quota, saas) diff --git a/orchestra/contrib/saas/backends/owncloud.py b/orchestra/contrib/saas/backends/owncloud.py new file mode 100644 index 00000000..a6496a42 --- /dev/null +++ b/orchestra/contrib/saas/backends/owncloud.py @@ -0,0 +1,175 @@ +import re +import sys +import textwrap +import time +import xml.etree.ElementTree as ET +from urllib.parse import urlparse + +import requests +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor + +from . import ApacheTrafficByName +from .. import settings + + +class OwnClouwAPIMixin(object): + def validate_response(self, response): + request = response.request + context = (request.method, response.url, request.body, response.status_code) + sys.stderr.write("%s %s '%s' HTTP %s\n" % context) + if response.status_code != requests.codes.ok: + raise RuntimeError("%s %s '%s' HTTP %s" % context) + root = ET.fromstring(response.text) + statuscode = root.find("./meta/statuscode").text + if statuscode != '100': + message = root.find("./meta/status").text + request = response.request + context = (request.method, response.url, request.body, statuscode, message) + raise RuntimeError("%s %s '%s' ERROR %s, %s" % context) + + def api_call(self, action, url_path, *args, **kwargs): + BASE_URL = settings.SAAS_OWNCLOUD_API_URL.rstrip('/') + url = '/'.join((BASE_URL, url_path)) + response = action(url, *args, **kwargs) + self.validate_response(response) + return response + + def api_get(self, url_path, *args, **kwargs): + return self.api_call(requests.get, url_path, *args, **kwargs) + + def api_post(self, url_path, *args, **kwargs): + return self.api_call(requests.post, url_path, *args, **kwargs) + + def api_put(self, url_path, *args, **kwargs): + return self.api_call(requests.put, url_path, *args, **kwargs) + + def api_delete(self, url_path, *args, **kwargs): + return self.api_call(requests.delete, url_path, *args, **kwargs) + + def create(self, saas): + data = { + 'userid': saas.name, + 'password': saas.password + } + self.api_post('users', data) + + def update(self, saas): + """ + key: email|quota|display|password + value: el valor a modificar. + Si es un email, tornarà un error si la direcció no te la "@" + Si es una quota, sembla que algo per l'estil "5G", "100M", etc. funciona. Quota 0 = infinit + "display" es el display name, no crec que el fem servir, és cosmetic + """ + data = { + 'key': 'password', + 'value': saas.password, + } + self.api_put('users/%s' % saas.name, data) + + def get_user(self, saas): + """ + { + 'displayname' + 'email' + 'quota' => + { + 'free' (en Bytes) + 'relative' (en tant per cent sense signe %, e.g. 68.17) + 'total' (en Bytes) + 'used' (en Bytes) + } + } + """ + response = self.api_get('users/%s' % saas.name) + root = ET.fromstring(response.text) + ret = {} + for data in root.find('./data'): + ret[data.tag] = data.text + ret['quota'] = {} + for data in root.find('.data/quota'): + ret['quota'][data.tag] = data.text + return ret + + +class OwnCloudController(OwnClouwAPIMixin, ServiceController): + """ + Creates a wordpress site on a WordPress MultiSite installation. + + You should point it to the database server + """ + verbose_name = _("ownCloud SaaS") + model = 'saas.SaaS' + default_route_match = "saas.service == 'owncloud'" + doc_settings = (settings, + ('SAAS_OWNCLOUD_API_URL',) + ) + + def update_or_create(self, saas, server): + try: + self.api_get('users/%s' % saas.name) + except RuntimeError: + if getattr(saas, 'password'): + self.create(saas) + else: + raise + else: + if getattr(saas, 'password'): + self.update(saas) + + def remove(self, saas, server): + self.api_delete('users/%s' % saas.name) + + def save(self, saas): + # TODO disable user https://github.com/owncloud/core/issues/12601 + self.append(self.update_or_create, saas) + + def delete(self, saas): + self.append(self.remove, saas) + + +class OwncloudTraffic(ApacheTrafficByName): + __doc__ = ApacheTrafficByName.__doc__ + verbose_name = _("ownCloud SaaS Traffic") + default_route_match = "saas.service == 'owncloud'" + doc_settings = (settings, + ('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_OWNCLOUD_LOG_PATH') + ) + log_path = settings.SAAS_OWNCLOUD_LOG_PATH + + +class OwnCloudDiskQuota(OwnClouwAPIMixin, ServiceMonitor): + model = 'saas.SaaS' + verbose_name = _("ownCloud SaaS Disk Quota") + default_route_match = "saas.service == 'owncloud'" + resource = ServiceMonitor.DISK + delete_old_equal_values = True + + def monitor(self, user): + context = self.get_context(user) + self.append("echo %(object_id)s $(monitor %(base_home)s)" % context) + + def get_context(self, user): + context = { + 'object_id': user.pk, + 'base_home': user.get_base_home(), + } + return replace(context, "'", '"') + + def get_quota(self, saas, server): + try: + user = self.get_user(saas) + except requests.exceptions.ConnectionError: + time.sleep(2) + user = self.get_user(saas) + context = { + 'object_id': saas.pk, + 'used': int(user['quota'].get('used', 0)), + } + sys.stdout.write('%(object_id)i %(used)i\n' % context) + + def monitor(self, saas): + self.append(self.get_quota, saas) diff --git a/orchestra/contrib/saas/backends/phplist.py b/orchestra/contrib/saas/backends/phplist.py new file mode 100644 index 00000000..2b2992e3 --- /dev/null +++ b/orchestra/contrib/saas/backends/phplist.py @@ -0,0 +1,239 @@ +import hashlib +import re +import sys +import textwrap + +import requests +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor +from orchestra.utils.sys import sshrun + +from .. import settings + + +class PhpListSaaSController(ServiceController): + """ + Creates a new phplist instance on a phpList multisite installation. + The site is created by means of creating a new database per phpList site, + but all sites share the same code. + + Different databases are used instead of prefixes because php-list reacts by launching + the installation process. + + // config/config.php + $site = getenv("SITE"); + if ( $site == '' ) { + $site = array_shift((explode(".",$_SERVER['HTTP_HOST']))); + } + $database_name = "phplist_mu_{$site}"; + """ + verbose_name = _("phpList SaaS") + model = 'saas.SaaS' + default_route_match = "saas.service == 'phplist'" + serialize = True + + def error(self, msg): + sys.stderr.write(msg + '\n') + raise RuntimeError(msg) + + def _install_or_change_password(self, saas, server): + """ configures the database for the new site through HTTP to /admin/ """ + admin_link = 'https://%s/admin/' % saas.get_site_domain() + sys.stdout.write('admin_link: %s\n' % admin_link) + admin_content = requests.get(admin_link, verify=settings.SAAS_PHPLIST_VERIFY_SSL) + admin_content = admin_content.content.decode('utf8') + if admin_content.startswith('Cannot connect to Database'): + self.error("Database is not yet configured.") + install = re.search(r'([^"]+firstinstall[^"]+)', admin_content) + if install: + if not hasattr(saas, 'password'): + self.error("Password is missing.") + install_path = install.groups()[0] + install_link = admin_link + install_path[1:] + post = { + 'adminname': saas.name, + 'orgname': saas.account.username, + 'adminemail': saas.account.username, + 'adminpassword': saas.password, + } + response = requests.post( + install_link, data=post, verify=settings.SAAS_PHPLIST_VERIFY_SSL) + sys.stdout.write(response.content.decode('utf8')+'\n') + if response.status_code != 200: + self.error("Bad status code %i." % response.status_code) + else: + md5_password = hashlib.md5() + md5_password.update(saas.password.encode('ascii')) + context = self.get_context(saas) + context['digest'] = md5_password.hexdigest() + cmd = textwrap.dedent("""\ + mysql \\ + --host=%(db_host)s \\ + --user=%(db_user)s \\ + --password=%(db_pass)s \\ + --execute='UPDATE phplist_admin SET password="%(digest)s" where ID=1; \\ + UPDATE phplist_user_user SET password="%(digest)s" where ID=1;' \\ + %(db_name)s""") % context + sys.stdout.write('cmd: %s\n' % cmd) + sshrun(server.get_address(), cmd, persist=True) + + def save(self, saas): + if hasattr(saas, 'password'): + self.append(self._install_or_change_password, saas) + context = self.get_context(saas) + if context['crontab']: + context['escaped_crontab'] = context['crontab'].replace('$', '\\$') + self.append(textwrap.dedent("""\ + # Configuring phpList crontabs + if ! crontab -u %(user)s -l | grep 'phpList:"%(site_name)s"' > /dev/null; then + cat << EOF | su - %(user)s --shell /bin/bash -c 'crontab' + $(crontab -u %(user)s -l) + + # %(banner)s - phpList:"%(site_name)s" + %(escaped_crontab)s + EOF + fi""") % context + ) + + def delete(self, saas): + context = self.get_context(saas) + if context['crontab']: + context['crontab_regex'] = '\\|'.join(context['crontab'].splitlines()) + context['crontab_regex'] = context['crontab_regex'].replace('*', '\\*') + self.append(textwrap.dedent("""\ + crontab -u %(user)s -l \\ + | grep -v 'phpList:"%(site_name)s"\\|%(crontab_regex)s' \\ + | su - %(user)s --shell /bin/bash -c 'crontab' + """) % context + ) + + def get_context(self, saas): + context = { + 'banner': self.get_banner(), + 'name': saas.name, + 'site_name': saas.name, + 'phplist_path': settings.SAAS_PHPLIST_PATH, + 'user': settings.SAAS_PHPLIST_SYSTEMUSER, + 'db_user': settings.SAAS_PHPLIST_DB_USER, + 'db_pass': settings.SAAS_PHPLIST_DB_PASS, + 'db_name': settings.SAAS_PHPLIST_DB_NAME, + 'db_host': settings.SAAS_PHPLIST_DB_HOST, + } + context.update({ + 'crontab': settings.SAAS_PHPLIST_CRONTAB % context, + 'db_name': context['db_name'] % context, + }) + return context + + +class PhpListTraffic(ServiceMonitor): + verbose_name = _("phpList SaaS Traffic") + model = 'saas.SaaS' + default_route_match = "saas.service == 'phplist'" + resource = ServiceMonitor.TRAFFIC + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + doc_settings = (settings, + ('SAAS_PHPLIST_MAIL_LOG_PATH',) + ) + + def prepare(self): + mail_log = settings.SAAS_PHPLIST_MAIL_LOG_PATH + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'mail_logs': str((mail_log, mail_log+'.1')), + } + self.append(textwrap.dedent("""\ + import sys + from datetime import datetime + from dateutil import tz + + def prepare(object_id, list_domain, ini_date): + global lists + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + lists[list_domain] = [ini_date, object_id, 0] + + def inside_period(month, day, time, ini_date): + global months + global end_datetime + # Mar 9 17:13:22 + month = months[month] + year = end_datetime.year + if month == '12' and end_datetime.month == 1: + year = year+1 + if len(day) == 1: + day = '0' + day + date = str(year) + month + day + date += time.replace(':', '') + return ini_date < int(date) < end_date + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + # Converts orchestra's UTC dates to local timezone + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + maillogs = {mail_logs} + end_datetime = to_local_timezone('{current_date}') + end_date = int(end_datetime.strftime('%Y%m%d%H%M%S')) + months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') + months = dict((m, '%02d' % n) for n, m in enumerate(months, 1)) + + lists = {{}} + id_to_domain = {{}} + + def monitor(lists, id_to_domain, maillogs): + for maillog in maillogs: + try: + with open(maillog, 'r') as maillog: + for line in maillog.readlines(): + if ': message-id=<' in line: + # Sep 15 09:36:51 web postfix/cleanup[8138]: C20FF244283: message-id= + month, day, time, __, __, id, message_id = line.split()[:7] + list_domain = message_id.split('@')[1][:-1] + try: + opts = lists[list_domain] + except KeyError: + pass + else: + ini_date = opts[0] + if inside_period(month, day, time, ini_date): + id = id[:-1] + id_to_domain[id] = list_domain + elif '>, size=' in line: + # Sep 15 09:36:51 web postfix/qmgr[2296]: C20FF244283: from=, size=12252, nrcpt=1 (queue active) + month, day, time, __, __, id, __, size = line.split()[:8] + id = id[:-1] + try: + list_domain = id_to_domain[id] + except KeyError: + pass + else: + opts = lists[list_domain] + size = int(size[5:-1]) + opts[2] += size + except IOError as e: + sys.stderr.write(str(e)+'\\n') + for opts in lists.values(): + print opts[1], opts[2] + """).format(**context) + ) + + def commit(self): + self.append('monitor(lists, id_to_domain, maillogs)') + + def monitor(self, saas): + context = self.get_context(saas) + self.append("prepare(%(object_id)s, '%(list_domain)s', '%(last_date)s')" % context) + + def get_context(self, saas): + context = { + 'list_domain': saas.get_site_domain(), + 'object_id': saas.pk, + 'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + return context diff --git a/orchestra/contrib/saas/backends/wordpressmu.py b/orchestra/contrib/saas/backends/wordpressmu.py new file mode 100644 index 00000000..7a8e75d8 --- /dev/null +++ b/orchestra/contrib/saas/backends/wordpressmu.py @@ -0,0 +1,279 @@ +import re +import sys +import textwrap +import time +from functools import partial +from urllib.parse import urlparse + +import requests +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController + +from . import ApacheTrafficByHost +from .. import settings + + +class WordpressMuController(ServiceController): + """ + Creates a wordpress site on a WordPress MultiSite installation. + + You should point it to the database server + """ + verbose_name = _("Wordpress multisite") + model = 'saas.SaaS' + default_route_match = "saas.service == 'wordpress'" + doc_settings = (settings, + ('SAAS_WORDPRESS_ADMIN_PASSWORD', 'SAAS_WORDPRESS_MAIN_URL', 'SAAS_WORDPRESS_VERIFY_SSL') + ) + VERIFY = settings.SAAS_WORDPRESS_VERIFY_SSL + + def with_retry(self, method, *args, retries=1, sleep=0.5, **kwargs): + for i in range(retries): + try: + return method(*args, verify=self.VERIFY, **kwargs) + except requests.exceptions.ConnectionError: + if i >= retries: + raise + sys.stderr.write("Connection error while {method}{args}, retry {i}/{retries}\n".format( + method=method.__name__, args=str(args), i=i, retries=retries)) + time.sleep(sleep) + + def login(self, session): + main_url = self.get_main_url() + login_url = main_url + '/wp-login.php' + login_data = { + 'log': 'admin', + 'pwd': settings.SAAS_WORDPRESS_ADMIN_PASSWORD, + 'redirect_to': '/wp-admin/' + } + sys.stdout.write("Login URL: %s\n" % login_url) + response = self.with_retry(session.post, login_url, data=login_data) + if response.url != main_url + '/wp-admin/': + raise IOError("Failure login to remote application (%s)" % login_url) + + def get_main_url(self): + main_url = settings.SAAS_WORDPRESS_MAIN_URL + return main_url.rstrip('/') + + def validate_response(self, response): + if response.status_code != 200: + content = response.content.decode('utf8') + errors = re.findall(r'\n\t

    (.*)

    ', content) + raise RuntimeError(errors[0] if errors else 'Unknown %i error' % response.status_code) + + def get_id(self, session, saas): + blog_id = saas.data.get('blog_id') + search = self.get_main_url() + search += '/wp-admin/network/sites.php?s=%s&action=blogs' % saas.name + regex = re.compile( + '%s' % saas.name + ) + sys.stdout.write("Search URL: %s\n" % search) + response = self.with_retry(session.get, search) + content = response.content.decode('utf8') + # Get id + ids = regex.search(content) + if not ids and not blog_id: + raise RuntimeError("Blog '%s' not found" % saas.name) + if ids: + ids = ids.groups() + if len(ids) > 1 and not blog_id: + raise ValueError("Multiple matches") + return blog_id or int(ids[0]), content + + def create_blog(self, saas, server): + if saas.data.get('blog_id'): + return + + session = requests.Session() + self.login(session) + + # Check if blog already exists + try: + blog_id, content = self.get_id(session, saas) + except RuntimeError: + url = self.get_main_url() + url += '/wp-admin/network/site-new.php' + sys.stdout.write("Create URL: %s\n" % url) + content = self.with_retry(session.get, url).content.decode('utf8') + + wpnonce = re.compile('name="_wpnonce_add-blog"\s+value="([^"]*)"') + try: + wpnonce = wpnonce.search(content).groups()[0] + except AttributeError: + raise RuntimeError("wpnonce not foud in %s" % content) + + url += '?action=add-site' + data = { + 'blog[domain]': saas.name, + 'blog[title]': saas.name, + 'blog[email]': saas.account.email, + '_wpnonce_add-blog': wpnonce, + } + + # Validate response + response = self.with_retry(session.post, url, data=data) + self.validate_response(response) + blog_id = re.compile(r'') + content = response.content.decode('utf8') + blog_id = blog_id.search(content).groups()[0] + sys.stdout.write("Created blog ID: %s\n" % blog_id) + saas.data['blog_id'] = int(blog_id) + saas.save(update_fields=('data',)) + return True + else: + sys.stdout.write("Retrieved blog ID: %s\n" % blog_id) + saas.data['blog_id'] = int(blog_id) + saas.save(update_fields=('data',)) + + def do_action(self, action, session, id, content, saas): + url_regex = r"""]*)['"]>""" % action + action_url = re.search(url_regex, content).groups()[0].replace("&", '&') + sys.stdout.write("%s confirm URL: %s\n" % (action, action_url)) + + content = self.with_retry(session.get, action_url).content.decode('utf8') + wpnonce = re.compile('name="_wpnonce"\s+value="([^"]*)"') + try: + wpnonce = wpnonce.search(content).groups()[0] + except AttributeError: + raise RuntimeError(re.search(r'([^<]+)<', content).groups()[0]) + data = { + 'action': action, + 'id': id, + '_wpnonce': wpnonce, + '_wp_http_referer': '/wp-admin/network/sites.php', + } + action_url = self.get_main_url() + action_url += '/wp-admin/network/sites.php?action=%sblog' % action + sys.stdout.write("%s URL: %s\n" % (action, action_url)) + response = self.with_retry(session.post, action_url, data=data) + self.validate_response(response) + + def is_active(self, content): + return bool( + re.findall(r"""Warning: ' + 'Related website directive does not exist for %s URL !' % + self.instance.custom_url) + else: + url = change_url(website) + link = '
    Related website:     %s' % (url, website.name) + self.fields['custom_url'].help_text += link + else: + site_domain = self.plugin.site_domain + context = { + 'site_name': '<site_name>', + 'name': '<site_name>', + } + site_domain = site_domain % context + if '<site_name>' in site_domain: + site_link = site_domain + else: + site_link = '%s' % (site_domain, site_domain) + self.fields['site_url'].widget.display = site_link + self.fields['name'].label = _("Site name") if '%(' in self.plugin.site_domain else _("Username") + + +class SaaSPasswordForm(SaaSBaseForm): + password = forms.CharField(label=_("Password"), required=False, + widget=SpanWidget(display='Unknown password'), + validators=[ + validators.validate_password, + RegexValidator(r'^[^"\'\\]+$', + _('Enter a valid password. ' + 'This value may contain any ascii character except for ' + ' \'/"/\\/ characters.'), 'invalid'), + ], + help_text=_("Passwords are not stored, so there is no way to see this " + "service's password, but you can change the password using " + "this form.")) + password1 = forms.CharField(label=_("Password"), + widget=forms.PasswordInput(attrs={'autocomplete': 'off'}), + validators=[validators.validate_password]) + password2 = forms.CharField(label=_("Password confirmation"), + widget=forms.PasswordInput, + help_text=_("Enter the same password as above, for verification.")) + + def __init__(self, *args, **kwargs): + super(SaaSPasswordForm, self).__init__(*args, **kwargs) + if self.is_change: + self.fields['password1'].required = False + self.fields['password1'].widget = forms.HiddenInput() + self.fields['password2'].required = False + self.fields['password2'].widget = forms.HiddenInput() + else: + self.fields['password'].widget = forms.HiddenInput() + self.fields['password1'].help_text = _("Suggestion: %s") % random_ascii(10) + + def clean_password2(self): + if not self.is_change: + password1 = self.cleaned_data.get("password1") + password2 = self.cleaned_data.get("password2") + if password1 and password2 and password1 != password2: + msg = _("The two password fields didn't match.") + raise forms.ValidationError(msg) + return password2 + + def save(self, commit=True): + obj = super(SaaSPasswordForm, self).save(commit=commit) + if not self.is_change: + obj.set_password(self.cleaned_data["password1"]) + return obj diff --git a/orchestra/contrib/saas/models.py b/orchestra/contrib/saas/models.py new file mode 100644 index 00000000..967bf7b8 --- /dev/null +++ b/orchestra/contrib/saas/models.py @@ -0,0 +1,87 @@ +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ +from jsonfield import JSONField + +from orchestra.core import validators + +from .fields import VirtualDatabaseRelation +from .services import SoftwareService + + +class SaaSQuerySet(models.QuerySet): + def create(self, **kwargs): + """ Sets password if provided, all within a single DB operation """ + password = kwargs.pop('password') + saas = SaaS(**kwargs) + if password: + saas.set_password(password) + saas.save() + return saas + + +class SaaS(models.Model): + service = models.CharField(_("service"), max_length=32, + choices=SoftwareService.get_choices()) + name = models.CharField(_("Name"), max_length=64, + help_text=_("Required. 64 characters or fewer. Letters, digits and ./- only."), + validators=[validators.validate_hostname]) + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("account"), related_name='saas') + is_active = models.BooleanField(_("active"), default=True, + help_text=_("Designates whether this service should be treated as active. ")) + data = JSONField(_("data"), default={}, + help_text=_("Extra information dependent of each service.")) + custom_url = models.URLField(_("custom URL"), blank=True, + help_text=_("Optional and alternative URL for accessing this service instance. " + "i.e. https://wiki.mydomain/doku/
    " + "A related website will be automatically configured if needed.")) + database = models.ForeignKey('databases.Database', + on_delete=models.SET_NULL, null=True, blank=True) + + # Some SaaS sites may need a database, with this virtual field we tell the ORM to delete them + databases = VirtualDatabaseRelation('databases.Database') + objects = SaaSQuerySet.as_manager() + + class Meta: + verbose_name = "SaaS" + verbose_name_plural = "SaaS" + unique_together = ( + ('name', 'service'), + ) + + def __str__(self): + return "%s@%s" % (self.name, self.service) + + @cached_property + def service_class(self): + return SoftwareService.get(self.service) + + @cached_property + def service_instance(self): + """ Per request lived service_instance """ + return self.service_class(self) + + @cached_property + def active(self): + return self.is_active and self.account.is_active + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = True + self.save(update_fields=('is_active',)) + + def clean(self): + if not self.pk: + self.name = self.name.lower() + self.service_instance.clean() + self.data = self.service_instance.clean_data() + + def get_site_domain(self): + return self.service_instance.get_site_domain() + + def set_password(self, password): + self.password = password diff --git a/orchestra/contrib/saas/serializers.py b/orchestra/contrib/saas/serializers.py new file mode 100644 index 00000000..a0da1a17 --- /dev/null +++ b/orchestra/contrib/saas/serializers.py @@ -0,0 +1,28 @@ +from django.forms import widgets +from django.core.validators import RegexValidator +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.api.serializers import SetPasswordHyperlinkedSerializer +from orchestra.contrib.accounts.serializers import AccountSerializerMixin +from orchestra.core import validators + +from .models import SaaS + + +class SaaSSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): + data = serializers.DictField(required=False) + password = serializers.CharField(write_only=True, required=False, + style={'widget': widgets.PasswordInput}, + validators=[ + validators.validate_password, + RegexValidator(r'^[^"\'\\]+$', + _('Enter a valid password. ' + 'This value may contain any ascii character except for ' + ' \'/"/\\/ characters.'), 'invalid'), + ]) + + class Meta: + model = SaaS + fields = ('url', 'id', 'name', 'service', 'is_active', 'data', 'password') + postonly_fields = ('name', 'service', 'password') diff --git a/orchestra/contrib/saas/services/__init__.py b/orchestra/contrib/saas/services/__init__.py new file mode 100644 index 00000000..4720b7c2 --- /dev/null +++ b/orchestra/contrib/saas/services/__init__.py @@ -0,0 +1 @@ +from .options import SoftwareService diff --git a/orchestra/contrib/saas/services/bscw.py b/orchestra/contrib/saas/services/bscw.py new file mode 100644 index 00000000..7fef1cda --- /dev/null +++ b/orchestra/contrib/saas/services/bscw.py @@ -0,0 +1,25 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from .. import settings +from ..forms import SaaSPasswordForm +from .options import SoftwareService + + +class BSCWForm(SaaSPasswordForm): + email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size': '40'})) + + +class BSCWDataSerializer(serializers.Serializer): + email = serializers.EmailField(label=_("Email")) + + +class BSCWService(SoftwareService): + name = 'bscw' + verbose_name = "BSCW" + form = BSCWForm + serializer = BSCWDataSerializer + icon = 'orchestra/icons/apps/BSCW.png' + site_domain = settings.SAAS_BSCW_DOMAIN + change_readonly_fields = ('email',) diff --git a/orchestra/contrib/saas/services/dokuwiki.py b/orchestra/contrib/saas/services/dokuwiki.py new file mode 100644 index 00000000..5195f848 --- /dev/null +++ b/orchestra/contrib/saas/services/dokuwiki.py @@ -0,0 +1,24 @@ +from urllib.parse import urlparse + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from .options import SoftwareService +from .. import settings + + +class DokuWikiService(SoftwareService): + name = 'dokuwiki' + verbose_name = "Dowkuwiki" + icon = 'orchestra/icons/apps/Dokuwiki.png' + site_domain = settings.SAAS_DOKUWIKI_DOMAIN + allow_custom_url = settings.SAAS_DOKUWIKI_ALLOW_CUSTOM_URL + + def clean(self): + if self.allow_custom_url and self.instance.custom_url: + url = urlparse(self.instance.custom_url) + if url.path and url.path != '/': + raise ValidationError({ + 'custom_url': _("Support for specific URL paths (%s) is not implemented.") % url.path + }) + super(DokuWikiService, self).clean() diff --git a/orchestra/contrib/saas/services/drupal.py b/orchestra/contrib/saas/services/drupal.py new file mode 100644 index 00000000..e40291ed --- /dev/null +++ b/orchestra/contrib/saas/services/drupal.py @@ -0,0 +1,10 @@ +from .options import SoftwareService + +from .. import settings + + +class DrupalService(SoftwareService): + name = 'drupal' + verbose_name = "Drupal" + icon = 'orchestra/icons/apps/Drupal.png' + site_domain = settings.SAAS_DRUPAL_DOMAIN diff --git a/orchestra/contrib/saas/services/gitlab.py b/orchestra/contrib/saas/services/gitlab.py new file mode 100644 index 00000000..b8d62cfa --- /dev/null +++ b/orchestra/contrib/saas/services/gitlab.py @@ -0,0 +1,35 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.forms import widgets + +from .. import settings +from ..forms import SaaSPasswordForm +from .options import SoftwareService + + +class GitLabForm(SaaSPasswordForm): + email = forms.EmailField(label=_("Email"), + help_text=_("Initial email address, changes on the GitLab server are not reflected here.")) + + +class GitLaChangeForm(GitLabForm): + user_id = forms.IntegerField(label=("User ID"), widget=widgets.SpanWidget, + help_text=_("ID of this user used by GitLab, the only attribute that doesn't change.")) + + +class GitLabSerializer(serializers.Serializer): + email = serializers.EmailField(label=_("Email")) + user_id = serializers.IntegerField(label=_("User ID"), allow_null=True, required=False) + + +class GitLabService(SoftwareService): + name = 'gitlab' + form = GitLabForm + change_form = GitLaChangeForm + serializer = GitLabSerializer + site_domain = settings.SAAS_GITLAB_DOMAIN + change_readonly_fields = ('email', 'user_id',) + verbose_name = "GitLab" + icon = 'orchestra/icons/apps/gitlab.png' diff --git a/orchestra/contrib/saas/services/helpers.py b/orchestra/contrib/saas/services/helpers.py new file mode 100644 index 00000000..0deb36f5 --- /dev/null +++ b/orchestra/contrib/saas/services/helpers.py @@ -0,0 +1,134 @@ +from collections import defaultdict +from urllib.parse import urlparse + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.websites.models import Website, WebsiteDirective, Content +from orchestra.contrib.websites.validators import validate_domain_protocol +from orchestra.contrib.orchestration.models import Server +from orchestra.utils.python import AttrDict + + +def full_clean(obj, exclude=None): + try: + obj.full_clean(exclude=exclude) + except ValidationError as e: + raise ValidationError({ + 'custom_url': _("Error validating related %s: %s") % (type(obj).__name__, e), + }) + + +def clean_custom_url(saas): + instance = saas.instance + instance.custom_url = instance.custom_url.strip() + url = urlparse(instance.custom_url) + if not url.path: + instance.custom_url += '/' + url = urlparse(instance.custom_url) + try: + protocol, valid_protocols = saas.PROTOCOL_MAP[url.scheme] + except KeyError: + raise ValidationError({ + 'custom_url': _("%s scheme not supported (http/https)") % url.scheme, + }) + account = instance.account + # get or create website + try: + website = Website.objects.get( + protocol__in=valid_protocols, + domains__name=url.netloc, + account=account, + ) + except Website.DoesNotExist: + # get or create domain + Domain = Website.domains.field.related_model + try: + domain = Domain.objects.get(name=url.netloc) + except Domain.DoesNotExist: + raise ValidationError({ + 'custom_url': _("Domain %s does not exist.") % url.netloc, + }) + if domain.account != account: + raise ValidationError({ + 'custom_url': _("Domain %s does not belong to account %s, it's from %s.") % + (url.netloc, account, domain.account), + }) + # Create new website for custom_url + # Changed by daniel: hardcode target_server to web.pangea.lan, consider putting it into settings.py + tgt_server = Server.objects.get(name='web.pangea.lan') + website = Website(name=url.netloc, protocol=protocol, account=account, target_server=tgt_server) + full_clean(website) + try: + validate_domain_protocol(website, domain, protocol) + except ValidationError as e: + raise ValidationError({ + 'custom_url': _("Error validating related %s: %s") % (type(website).__name__, e), + }) + # get or create directive + try: + directive = website.directives.get(name=saas.get_directive_name()) + except WebsiteDirective.DoesNotExist: + directive = WebsiteDirective(name=saas.get_directive_name(), value=url.path) + if not directive.pk or directive.value != url.path: + directive.value = url.path + if website.pk: + directive.website = website + full_clean(directive) + # Adaptation of orchestra.websites.forms.WebsiteDirectiveInlineFormSet.clean() + locations = set( + Content.objects.filter(website=website).values_list('path', flat=True) + ) + values = defaultdict(list) + directives = WebsiteDirective.objects.filter(website=website) + for wdirective in directives.exclude(pk=directive.pk): + fdirective = AttrDict({ + 'name': wdirective.name, + 'value': wdirective.value + }) + try: + wdirective.directive_instance.validate_uniqueness(fdirective, values, locations) + except ValidationError as err: + raise ValidationError({ + 'custom_url': _("Another directive with this URL path exists (%s)." % err) + }) + else: + full_clean(directive, exclude=('website',)) + return directive + + +def create_or_update_directive(saas): + instance = saas.instance + url = urlparse(instance.custom_url) + protocol, valid_protocols = saas.PROTOCOL_MAP[url.scheme] + account = instance.account + # get or create website + try: + website = Website.objects.get( + protocol__in=valid_protocols, + domains__name=url.netloc, + account=account, + ) + except Website.DoesNotExist: + Domain = Website.domains.field.related_model + domain = Domain.objects.get(name=url.netloc) + # Create new website for custom_url + tgt_server = Server.objects.get(name='web.pangea.lan') + website = Website(name=url.netloc, protocol=protocol, account=account, target_server=tgt_server) + website.save() + website.domains.add(domain) + # get or create directive + try: + directive = website.directives.get(name=saas.get_directive_name()) + except WebsiteDirective.DoesNotExist: + directive = WebsiteDirective(name=saas.get_directive_name(), value=url.path) + if not directive.pk or directive.value != url.path: + directive.value = url.path + directive.website = website + directive.save() + return directive + + +def update_directive(saas): + saas.instance.custom_url = saas.instance.custom_url.strip() + url = urlparse(saas.instance.custom_url) diff --git a/orchestra/contrib/saas/services/moodle.py b/orchestra/contrib/saas/services/moodle.py new file mode 100644 index 00000000..2b6580e4 --- /dev/null +++ b/orchestra/contrib/saas/services/moodle.py @@ -0,0 +1,25 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +from orchestra.forms.widgets import SpanWidget + +from .. import settings +from ..forms import SaaSPasswordForm +from .options import SoftwareService + + +class MoodleForm(SaaSPasswordForm): + admin_username = forms.CharField(label=_("Admin username"), required=False, + widget=SpanWidget(display='admin')) + + +class MoodleService(SoftwareService): + name = 'moodle' + verbose_name = "Moodle" + form = MoodleForm + description_field = 'site_name' + icon = 'orchestra/icons/apps/Moodle.png' + site_domain = settings.SAAS_MOODLE_DOMAIN + allow_custom_url = settings.SAAS_MOODLE_ALLOW_CUSTOM_URL + db_name = settings.SAAS_MOODLE_DB_NAME + db_user = settings.SAAS_MOODLE_DB_USER diff --git a/orchestra/contrib/saas/services/nextcloud.py b/orchestra/contrib/saas/services/nextcloud.py new file mode 100644 index 00000000..2027352e --- /dev/null +++ b/orchestra/contrib/saas/services/nextcloud.py @@ -0,0 +1,13 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from .. import settings +from .options import SoftwareService + + +class NextCloudService(SoftwareService): + name = 'nextcloud' + verbose_name = "nextCloud" + icon = 'orchestra/icons/apps/nextCloud.png' + site_domain = settings.SAAS_NEXTCLOUD_DOMAIN diff --git a/orchestra/contrib/saas/services/options.py b/orchestra/contrib/saas/services/options.py new file mode 100644 index 00000000..37413a89 --- /dev/null +++ b/orchestra/contrib/saas/services/options.py @@ -0,0 +1,205 @@ +import importlib +import os +from functools import lru_cache +from urllib.parse import urlparse + +from django.core.exceptions import ValidationError, ObjectDoesNotExist +from django.utils.translation import gettext_lazy as _ + +from orchestra import plugins +from orchestra.contrib.databases.models import Database, DatabaseUser +from orchestra.contrib.orchestration import Operation +from orchestra.contrib.websites.models import Website, WebsiteDirective +from orchestra.utils.apps import isinstalled +from orchestra.utils.functional import cached +from orchestra.utils.python import import_class + +from . import helpers +from .. import settings +from ..forms import SaaSPasswordForm + + +class SoftwareService(plugins.Plugin, metaclass=plugins.PluginMount): + PROTOCOL_MAP = { + 'http': (Website.HTTP, (Website.HTTP, Website.HTTP_AND_HTTPS)), + 'https': (Website.HTTPS_ONLY, (Website.HTTPS, Website.HTTP_AND_HTTPS, Website.HTTPS_ONLY)), + } + + name = None + verbose_name = None + form = SaaSPasswordForm + site_domain = None + has_custom_domain = False + icon = 'orchestra/icons/apps.png' + class_verbose_name = _("Software as a Service") + plugin_field = 'service' + allow_custom_url = False + + @classmethod + @lru_cache() + def get_plugins(cls, all=False): + if all: + for module in os.listdir(os.path.dirname(__file__)): + if module not in ('options.py', '__init__.py') and module[-3:] == '.py': + importlib.import_module('.'+module[:-3], __package__) + plugins = super().get_plugins() + else: + plugins = [] + for cls in settings.SAAS_ENABLED_SERVICES: + plugins.append(import_class(cls)) + return plugins + + def get_change_readonly_fields(cls): + fields = super(SoftwareService, cls).get_change_readonly_fields() + return fields + ('name',) + + def get_site_domain(self): + context = { + 'site_name': self.instance.name, + 'name': self.instance.name, + } + return self.site_domain % context + + def clean(self): + if self.allow_custom_url: + if self.instance.custom_url: + if isinstalled('orchestra.contrib.websites'): + helpers.clean_custom_url(self) + elif self.instance.custom_url: + raise ValidationError({ + 'custom_url': _("Custom URL not allowed for this service."), + }) + + def clean_data(self): + data = super(SoftwareService, self).clean_data() + if not self.instance.pk: + try: + log = Operation.execute_action(self.instance, 'validate_creation')[0] + except IndexError: + pass + else: + if log.state != log.SUCCESS: + raise ValidationError(_("Validate creation execution has failed.")) + errors = {} + if 'user-exists' in log.stdout: + errors['name'] = _("User with this username already exists.") + if 'email-exists' in log.stdout: + errors['email'] = _("User with this email address already exists.") + if errors: + raise ValidationError(errors) + return data + + def get_directive_name(self): + return '%s-saas' % self.name + + def get_directive(self, *args): + if not args: + instance = self.instance + else: + instance = args[0] + url = urlparse(instance.custom_url) + account = instance.account + return WebsiteDirective.objects.get( + name=self.get_directive_name(), + value=url.path, + website__protocol__in=self.PROTOCOL_MAP[url.scheme][1], + website__domains__name=url.netloc, + website__account=account, + ) + + def get_website(self): + url = urlparse(self.instance.custom_url) + account = self.instance.account + return Website.objects.get( + protocol__in=self.PROTOCOL_MAP[url.scheme][1], + domains__name=url.netloc, + account=account, + directives__name=self.get_directive_name(), + directives__value=url.path, + ) + + def create_or_update_directive(self): + return helpers.create_or_update_directive(self) + + def delete_directive(self): + directive = None + try: + old = type(self.instance).objects.get(pk=self.instance.pk) + if old.custom_url: + directive = self.get_directive(old) + except ObjectDoesNotExist: + return + if directive is not None: + directive.delete() + + def save(self): + # pre instance.save() + if isinstalled('orchestra.contrib.websites'): + if self.instance.custom_url: + self.create_or_update_directive() + elif self.instance.pk: + self.delete_directive() + + def delete(self): + if isinstalled('orchestra.contrib.websites'): + self.delete_directive() + + def get_related(self): + return [] + + +class DBSoftwareService(SoftwareService): + db_name = None + db_user = None + abstract = True + + def get_db_name(self): + context = { + 'name': self.instance.name, + 'site_name': self.instance.name, + } + db_name = self.db_name % context + # Limit for mysql database names + return db_name[:65] + + def get_db_user(self): + return self.db_user + + @cached + def get_account(self): + account_model = self.instance._meta.get_field('account') + return account_model.remote_field.model.objects.get_main() + + def validate(self): + super(DBSoftwareService, self).validate() + create = not self.instance.pk + if create: + account = self.get_account() + # Validated Database + db_user = self.get_db_user() + try: + DatabaseUser.objects.get(username=db_user) + except DatabaseUser.DoesNotExist: + raise ValidationError( + _("Global database user for PHPList '%(db_user)s' does not exists.") % { + 'db_user': db_user + } + ) + db = Database(name=self.get_db_name(), account=account) + try: + db.full_clean() + except ValidationError as e: + raise ValidationError({ + 'name': e.messages, + }) + + def save(self): + super(DBSoftwareService, self).save() + account = self.get_account() + # Database + db_name = self.get_db_name() + db_user = self.get_db_user() + db, db_created = account.databases.get_or_create(name=db_name, type=Database.MYSQL) + user = DatabaseUser.objects.get(username=db_user) + db.users.add(user) + self.instance.database_id = db.pk diff --git a/orchestra/contrib/saas/services/owncloud.py b/orchestra/contrib/saas/services/owncloud.py new file mode 100644 index 00000000..2a6d1210 --- /dev/null +++ b/orchestra/contrib/saas/services/owncloud.py @@ -0,0 +1,13 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from .. import settings +from .options import SoftwareService + + +class OwnCloudService(SoftwareService): + name = 'owncloud' + verbose_name = "ownCloud" + icon = 'orchestra/icons/apps/ownCloud.png' + site_domain = settings.SAAS_OWNCLOUD_DOMAIN diff --git a/orchestra/contrib/saas/services/phplist.py b/orchestra/contrib/saas/services/phplist.py new file mode 100644 index 00000000..9b97fe78 --- /dev/null +++ b/orchestra/contrib/saas/services/phplist.py @@ -0,0 +1,122 @@ +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.urls import reverse +from django.db.models import Q +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.mailboxes.models import Mailbox +from orchestra.forms.widgets import SpanWidget + +from .. import settings +from ..forms import SaaSPasswordForm +from .options import DBSoftwareService + + +class PHPListForm(SaaSPasswordForm): + admin_username = forms.CharField(label=_("Admin username"), required=False, + widget=SpanWidget(display='admin')) + database = forms.CharField(label=_("Database"), required=False, + help_text=_("Database dedicated to this phpList instance."), + widget=SpanWidget(display=settings.SAAS_PHPLIST_DB_NAME.replace( + '%(', '<').replace(')s', '>'))) + mailbox = forms.CharField(label=_("Bounces mailbox"), required=False, + help_text=_("Dedicated mailbox used for reciving bounces."), + widget=SpanWidget(display=settings.SAAS_PHPLIST_BOUNCES_MAILBOX_NAME.replace( + '%(', '<').replace(')s', '>'))) + + def __init__(self, *args, **kwargs): + super(PHPListForm, self).__init__(*args, **kwargs) + self.fields['name'].label = _("Site name") + context = { + 'site_name': '<site_name>', + 'name': '<site_name>', + } + domain = self.plugin.site_domain % context + help_text = _("Admin URL http://{}/admin/").format(domain) + self.fields['site_url'].help_text = help_text + validator = validators.MaxLengthValidator(settings.SAAS_PHPLIST_NAME_MAX_LENGTH) + self.fields['name'].validators.append(validator) + + +class PHPListChangeForm(PHPListForm): + def __init__(self, *args, **kwargs): + super(PHPListChangeForm, self).__init__(*args, **kwargs) + site_domain = self.instance.get_site_domain() + admin_url = "http://%s/admin/" % site_domain + help_text = _("Admin URL {0}").format(admin_url) + self.fields['site_url'].help_text = help_text + # DB link + db = self.instance.database + db_url = reverse('admin:databases_database_change', args=(db.pk,)) + db_link = mark_safe('%s' % (db_url, db.name)) + self.fields['database'].widget = SpanWidget(original=db.name, display=db_link) + # Mailbox link + mailbox_id = self.instance.data.get('mailbox_id') + if mailbox_id: + try: + mailbox = Mailbox.objects.get(id=mailbox_id) + except Mailbox.DoesNotExist: + pass + else: + mailbox_url = reverse('admin:mailboxes_mailbox_change', args=(mailbox.pk,)) + mailbox_link = mark_safe('%s' % (mailbox_url, mailbox.name)) + self.fields['mailbox'].widget = SpanWidget( + original=mailbox.name, display=mailbox_link) + + +class PHPListService(DBSoftwareService): + name = 'phplist' + verbose_name = "phpList" + form = PHPListForm + change_form = PHPListChangeForm + icon = 'orchestra/icons/apps/Phplist.png' + site_domain = settings.SAAS_PHPLIST_DOMAIN + allow_custom_url = settings.SAAS_PHPLIST_ALLOW_CUSTOM_URL + db_name = settings.SAAS_PHPLIST_DB_NAME + db_user = settings.SAAS_PHPLIST_DB_USER + + def get_mailbox_name(self): + context = { + 'name': self.instance.name, + 'site_name': self.instance.name, + } + return settings.SAAS_PHPLIST_BOUNCES_MAILBOX_NAME % context + + def validate(self): + super(PHPListService, self).validate() + create = not self.instance.pk + if create: + account = self.get_account() + # Validate mailbox + mailbox = Mailbox(name=self.get_mailbox_name(), account=account) + try: + mailbox.full_clean() + except ValidationError as e: + raise ValidationError({ + 'name': e.messages, + }) + + def save(self): + super(PHPListService, self).save() + account = self.get_account() + # Mailbox + mailbox_name = self.get_mailbox_name() + mailbox, mb_created = account.mailboxes.get_or_create(name=mailbox_name) + if mb_created: + mailbox.set_password(settings.SAAS_PHPLIST_BOUNCES_MAILBOX_PASSWORD) + mailbox.save(update_fields=('password',)) + self.instance.data.update({ + 'mailbox_id': mailbox.pk, + 'mailbox_name': mailbox_name, + }) + + def delete(self): + super(PHPListService, self).save() + account = self.get_account() + # delete Mailbox (database will be deleted by ORM's cascade behaviour + mailbox_name = self.instance.data.get('mailbox_name') or self.get_mailbox_name() + mailbox_id = self.instance.data.get('mailbox_id') + qs = Q(Q(name=mailbox_name) | Q(id=mailbox_id)) + account.mailboxes.filter(qs).delete() diff --git a/orchestra/contrib/saas/services/seafile.py b/orchestra/contrib/saas/services/seafile.py new file mode 100644 index 00000000..bc29a42d --- /dev/null +++ b/orchestra/contrib/saas/services/seafile.py @@ -0,0 +1,31 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from .. import settings +from ..forms import SaaSPasswordForm +from .options import SoftwareService + + +# TODO monitor quota since out of sync? + +class SeaFileForm(SaaSPasswordForm): + email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'})) + quota = forms.IntegerField(label=_("Quota"), initial=settings.SAAS_SEAFILE_DEFAULT_QUOTA, + help_text=_("Disk quota in MB.")) + + +class SeaFileDataSerializer(serializers.Serializer): + email = serializers.EmailField(label=_("Email")) + quota = serializers.IntegerField(label=_("Quota"), default=settings.SAAS_SEAFILE_DEFAULT_QUOTA, + help_text=_("Disk quota in MB.")) + + +class SeaFileService(SoftwareService): + name = 'seafile' + verbose_name = "SeaFile" + form = SeaFileForm + serializer = SeaFileDataSerializer + icon = 'orchestra/icons/apps/seafile.png' + site_domain = settings.SAAS_SEAFILE_DOMAIN + change_readonly_fields = ('email',) diff --git a/orchestra/contrib/saas/services/wordpress.py b/orchestra/contrib/saas/services/wordpress.py new file mode 100644 index 00000000..141bb488 --- /dev/null +++ b/orchestra/contrib/saas/services/wordpress.py @@ -0,0 +1,45 @@ +from django import forms +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.forms import widgets + +from .options import SoftwareService +from .. import settings +from ..forms import SaaSBaseForm + + +class WordPressForm(SaaSBaseForm): + email = forms.EmailField(label=_("Email"), + help_text=_("A new user will be created if the above email address is not in the database.
    " + "The username and password will be mailed to this email address.")) + + def __init__(self, *args, **kwargs): + super(WordPressForm, self).__init__(*args, **kwargs) + if self.is_change: + admin_url = 'http://%s/wp-admin/' % self.instance.get_site_domain() + help_text = 'Admin URL: {0}'.format(admin_url) + self.fields['site_url'].help_text = mark_safe(help_text) + + +class WordPressChangeForm(WordPressForm): + blog_id = forms.IntegerField(label=("Blog ID"), widget=widgets.SpanWidget, required=False, + help_text=_("ID of this blog used by WordPress, the only attribute that doesn't change.")) + + +class WordPressDataSerializer(serializers.Serializer): + email = serializers.EmailField(label=_("Email")) + blog_id = serializers.IntegerField(label=_("Blog ID"), allow_null=True, required=False) + + +class WordPressService(SoftwareService): + name = 'wordpress' + verbose_name = "WordPress" + form = WordPressForm + change_form = WordPressChangeForm + serializer = WordPressDataSerializer + icon = 'orchestra/icons/apps/WordPress.png' + change_readonly_fields = ('email', 'blog_id') + site_domain = settings.SAAS_WORDPRESS_DOMAIN + allow_custom_url = settings.SAAS_WORDPRESS_ALLOW_CUSTOM_URL diff --git a/orchestra/contrib/saas/settings.py b/orchestra/contrib/saas/settings.py new file mode 100644 index 00000000..837d1e1c --- /dev/null +++ b/orchestra/contrib/saas/settings.py @@ -0,0 +1,341 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting +from orchestra.core.validators import validate_ip_address +from orchestra.settings import ORCHESTRA_BASE_DOMAIN + +from . import validators +from .. import saas + + +SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES', + ( + 'orchestra.contrib.saas.services.moodle.MoodleService', + 'orchestra.contrib.saas.services.bscw.BSCWService', + 'orchestra.contrib.saas.services.gitlab.GitLabService', + 'orchestra.contrib.saas.services.phplist.PHPListService', + 'orchestra.contrib.saas.services.wordpress.WordPressService', + 'orchestra.contrib.saas.services.dokuwiki.DokuWikiService', + 'orchestra.contrib.saas.services.drupal.DrupalService', + 'orchestra.contrib.saas.services.owncloud.OwnCloudService', + 'orchestra.contrib.saas.services.nextcloud.NextCloudService', +# 'orchestra.contrib.saas.services.seafile.SeaFileService', + ), + # lazy loading + choices=lambda: ((s.get_class_path(), s.get_class_path()) for s in saas.services.SoftwareService.get_plugins(all=True)), + multiple=True, +) + + +SAAS_TRAFFIC_IGNORE_HOSTS = Setting('SAAS_TRAFFIC_IGNORE_HOSTS', + ('127.0.0.1',), + help_text=_("IP addresses to ignore during traffic accountability."), + validators=[lambda hosts: (validate_ip_address(host) for host in hosts)] +) + + +# WordPress + +SAAS_WORDPRESS_ALLOW_CUSTOM_URL = Setting('SAAS_WORDPRESS_ALLOW_CUSTOM_URL', + True, + help_text=_("Whether allow custom URL to be specified or not."), + validators=[validators.validate_website_saas_directives('wordpress-saas')], +) + +SAAS_WORDPRESS_LOG_PATH = Setting('SAAS_WORDPRESS_LOG_PATH', + '', + help_text=_('Filesystem path for the webserver access logs.
    ' + 'LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Host}i\"" host'), +) + +SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESS_ADMIN_PASSWORD', + 'secret' +) + +SAAS_WORDPRESS_MAIN_URL = Setting('SAAS_WORDPRESS_MAIN_URL', + 'https://blogs.{}/'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_WORDPRESS_DOMAIN = Setting('SAAS_WORDPRESS_DOMAIN', + '%(site_name)s.blogs.{}'.format(ORCHESTRA_BASE_DOMAIN), +) + +SAAS_WORDPRESS_DB_NAME = Setting('SAAS_WORDPRESS_DB_NAME', + 'wordpressmu', + help_text=_("Needed for domain mapping when SAAS_WORDPRESS_ALLOW_CUSTOM_URL is enabled."), +) + +SAAS_WORDPRESS_VERIFY_SSL = Setting('SAAS_WORDPRESS_VERIFY_SSL', + True, + help_text=_("Verify SSL certificate on the HTTP requests performed by the backend."), +) + + +# DokuWiki + +SAAS_DOKUWIKI_ALLOW_CUSTOM_URL = Setting('SAAS_DOKUWIKI_ALLOW_CUSTOM_URL', + True, + help_text=_("Whether allow custom URL to be specified or not."), + validators=[validators.validate_website_saas_directives('dokuwiki-saas')], +) + +SAAS_DOKUWIKI_TEMPLATE_PATH = Setting('SAAS_DOKUWIKI_TEMPLATE_PATH', + '/home/httpd/htdocs/wikifarm/template.tar.gz' +) + +SAAS_DOKUWIKI_FARM_PATH = Setting('WEBSITES_DOKUWIKI_FARM_PATH', + '/home/httpd/htdocs/wikifarm/farm' +) + +SAAS_DOKUWIKI_DOMAIN = Setting('SAAS_DOKUWIKI_DOMAIN', + '%(site_name)s.dokuwiki.{}'.format(ORCHESTRA_BASE_DOMAIN), +) + +SAAS_DOKUWIKI_TEMPLATE_PATH = Setting('SAAS_DOKUWIKI_TEMPLATE_PATH', + '/var/www/wikifarm/template.tar.gz', +) + +SAAS_DOKUWIKI_FARM_PATH = Setting('SAAS_DOKUWIKI_FARM_PATH', + '/var/www/wikifarm/farm' +) + +SAAS_DOKUWIKI_USER = Setting('SAAS_DOKUWIKI_USER', + 'orchestra' +) + +SAAS_DOKUWIKI_GROUP = Setting('SAAS_DOKUWIKI_GROUP', + 'orchestra' +) + +SAAS_DOKUWIKI_LOG_PATH = Setting('SAAS_DOKUWIKI_LOG_PATH', + '', +) + + +# Drupal + +SAAS_DRUPAL_ALLOW_CUSTOM_URL = Setting('SAAS_DRUPAL_ALLOW_CUSTOM_URL', + True, + help_text=_("Whether allow custom URL to be specified or not."), + validators=[validators.validate_website_saas_directives('drupal-saas')], +) + +SAAS_DRUPAL_SITES_PATH = Setting('WEBSITES_DRUPAL_SITES_PATH', + '/home/httpd/htdocs/drupal-mu/sites/%(site_name)s', +) + +SAAS_DRUPAL_DOMAIN = Setting('SAAS_DRUPAL_DOMAIN', + '%(site_name)s.drupal.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + + +# PhpList + +SAAS_PHPLIST_ALLOW_CUSTOM_URL = Setting('SAAS_PHPLIST_ALLOW_CUSTOM_URL', + False, + help_text=_("Whether allow custom URL to be specified or not."), + validators=[validators.validate_website_saas_directives('phplist-saas')], +) + +SAAS_PHPLIST_DB_USER = Setting('SAAS_PHPLIST_DB_USER', + 'phplist_mu', + help_text=_("Needed for password changing support."), +) + +SAAS_PHPLIST_DB_PASS = Setting('SAAS_PHPLIST_DB_PASS', + 'secret', + help_text=_("Needed for password changing support."), +) + +SAAS_PHPLIST_DB_NAME = Setting('SAAS_PHPLIST_DB_NAME', + 'phplist_mu_%(site_name)s', + help_text=_("Needed for password changing support."), +) + +SAAS_PHPLIST_DB_HOST = Setting('SAAS_PHPLIST_DB_HOST', + 'loclahost', + help_text=_("Needed for password changing support."), +) + +SAAS_PHPLIST_BOUNCES_MAILBOX_NAME = Setting('SAAS_PHPLIST_BOUNCES_MAILBOX_NAME', + '%(site_name)s-list-bounces', +) + +SAAS_PHPLIST_NAME_MAX_LENGTH = Setting('SAAS_PHPLIST_NAME_MAX_LENGTH', + 32-13, + help_text=_("Because of max system group name of the bounces mailbox is 32."), +) + +SAAS_PHPLIST_BOUNCES_MAILBOX_PASSWORD = Setting('SAAS_PHPLIST_BOUNCES_MAILBOX_PASSWORD', + 'secret', +) + +SAAS_PHPLIST_DOMAIN = Setting('SAAS_PHPLIST_DOMAIN', + '%(site_name)s.lists.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_PHPLIST_VERIFY_SSL = Setting('SAAS_PHPLIST_VERIFY_SSL', + True, + help_text=_("Verify SSL certificate on the HTTP requests performed by the backend."), +) + +SAAS_PHPLIST_PATH = Setting('SAAS_PHPLIST_PATH', + '/var/www/phplist-mu', + help_text=_("Filesystem path to the phpList source code installed on the server. " + "Used by SAAS_PHPLIST_CRONTAB.") +) + +SAAS_PHPLIST_SYSTEMUSER = Setting('SAAS_PHPLIST_SYSTEMUSER', + 'root', + help_text=_("System user running phpList on the server." + "Used by SAAS_PHPLIST_CRONTAB.") +) + +SAAS_PHPLIST_CRONTAB = Setting('SAAS_PHPLIST_CRONTAB', + ('*/10 * * * * PHPLIST=%(phplist_path)s; export SITE="%(site_name)s"; php $PHPLIST/admin/index.php -c $PHPLIST/config/config.php -p processqueue > /dev/null\n' + '*/40 * * * * PHPLIST=%(phplist_path)s; export SITE="%(site_name)s"; php $PHPLIST/admin/index.php -c $PHPLIST/config/config.php -p processbounces > /dev/null'), + help_text=_("processqueue and processbounce phpList cron execution. " + "Left blank if you don't want crontab to be configured") +) + +SAAS_PHPLIST_MAIL_LOG_PATH = Setting('SAAS_PHPLIST_MAIL_LOG_PATH', + '/var/log/mail.log', +) + + +# SeaFile + +SAAS_SEAFILE_DOMAIN = Setting('SAAS_SEAFILE_DOMAIN', + 'seafile.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_SEAFILE_DEFAULT_QUOTA = Setting('SAAS_SEAFILE_DEFAULT_QUOTA', + 50 +) + + +# ownCloud + +SAAS_OWNCLOUD_DOMAIN = Setting('SAAS_OWNCLOUD_DOMAIN', + 'owncloud.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_OWNCLOUD_API_URL = Setting('SAAS_OWNCLOUD_API_URL', + 'https://admin:secret@owncloud.{}/ocs/v1.php/cloud/'.format(ORCHESTRA_BASE_DOMAIN), +) + +SAAS_OWNCLOUD_LOG_PATH = Setting('SAAS_OWNCLOUD_LOG_PATH', + '', + help_text=_('Filesystem path for the webserver access logs.
    ' + 'LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Host}i\"" host'), +) + + +# nextCloud +SAAS_NEXTCLOUD_DOMAIN = Setting('SAAS_NEXTCLOUD_DOMAIN', + 'nextcloud.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_NEXTCLOUD_API_URL = Setting('SAAS_NEXTCLOUD_API_URL', + 'https://admin:secret@nextcloud.{}/ocs/v1.php/cloud'.format(ORCHESTRA_BASE_DOMAIN), +) + +SAAS_NEXTCLOUD_LOG_PATH = Setting('SAAS_NEXTCLOUD_LOG_PATH', + '', + help_text=_('Filesystem path for the webserver access logs.
    ' + 'LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Host}i\"" host'), +) + + +# BSCW + +SAAS_BSCW_DOMAIN = Setting('SAAS_BSCW_DOMAIN', + 'bscw.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_BSCW_DEFAULT_QUOTA = Setting('SAAS_BSCW_DEFAULT_QUOTA', + 50, +) + +SAAS_BSCW_BSADMIN_PATH = Setting('SAAS_BSCW_BSADMIN_PATH', + '/home/httpd/bscw/bin/bsadmin', +) + + +# GitLab + +SAAS_GITLAB_ROOT_PASSWORD = Setting('SAAS_GITLAB_ROOT_PASSWORD', + 'secret', +) + +SAAS_GITLAB_DOMAIN = Setting('SAAS_GITLAB_DOMAIN', + 'gitlab.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_GITLAB_VERIFY_SSL = Setting('SAAS_GITLAB_VERIFY_SSL', + True, +) + +# Moodle + +SAAS_MOODLE_ALLOW_CUSTOM_URL = Setting('SAAS_MOODLE_ALLOW_CUSTOM_URL', + True, + help_text=_("Whether allow custom URL to be specified or not."), + validators=[validators.validate_website_saas_directives('moodle-saas')], +) + +SAAS_MOODLE_DB_USER = Setting('SAAS_MOODLE_DB_USER', + 'moodle_mu', + help_text=_("Needed for password changing support."), +) + +SAAS_MOODLE_DB_PASS = Setting('SAAS_MOODLE_DB_PASS', + 'secret', + help_text=_("Needed for password changing support."), +) + +SAAS_MOODLE_DB_NAME = Setting('SAAS_MOODLE_DB_NAME', + 'moodle_mu', + help_text=_("Needed for password changing support."), +) + +SAAS_MOODLE_DB_HOST = Setting('SAAS_MOODLE_DB_HOST', + 'loclahost', + help_text=_("Needed for password changing support."), +) + +SAAS_MOODLE_DOMAIN = Setting('SAAS_MOODLE_DOMAIN', + '%(site_name)s.courses.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + +SAAS_MOODLE_PATH = Setting('SAAS_MOODLE_PATH', + '/var/www/moodle-mu', + help_text=_("Filesystem path to the Moodle source code installed on the server. " + "Used by SAAS_MOODLE_CRONTAB.") +) + +SAAS_MOODLE_DATA_PATH = Setting('SAAS_MOODLE_DATA_PATH', + '/var/moodledata/%(site_name)s', + help_text=_("Filesystem path to the Moodle source code installed on the server. " + "Used by SAAS_MOODLE_CRONTAB.") +) + +SAAS_MOODLE_SYSTEMUSER = Setting('SAAS_MOODLE_SYSTEMUSER', + 'root', + help_text=_("System user running Moodle on the server." + "Used by SAAS_MOODLE_CRONTAB.") +) + +SAAS_MOODLE_CRONTAB = Setting('SAAS_MOODLE_CRONTAB', + '*/15 * * * * export SITE="%(site_name)s"; php %(moodle_path)s/admin/cli/cron.php >/dev/null', + help_text=_("Left blank if you don't want crontab to be configured") +) diff --git a/orchestra/contrib/saas/signals.py b/orchestra/contrib/saas/signals.py new file mode 100644 index 00000000..c3354b1a --- /dev/null +++ b/orchestra/contrib/saas/signals.py @@ -0,0 +1,22 @@ +from django.db.models.signals import pre_save, pre_delete +from django.dispatch import receiver + +from .models import SaaS + + +# Admin bulk deletion doesn't call model.delete() +# So, signals are used instead of model method overriding + +@receiver(pre_save, sender=SaaS, dispatch_uid='saas.service.save') +def type_save(sender, *args, **kwargs): + instance = kwargs['instance'] + instance.service_instance.save() + + +@receiver(pre_delete, sender=SaaS, dispatch_uid='saas.service.delete') +def type_delete(sender, *args, **kwargs): + instance = kwargs['instance'] + try: + instance.service_instance.delete() + except KeyError: + pass diff --git a/orchestra/contrib/saas/validators.py b/orchestra/contrib/saas/validators.py new file mode 100644 index 00000000..ac21bb88 --- /dev/null +++ b/orchestra/contrib/saas/validators.py @@ -0,0 +1,14 @@ +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra.utils.apps import isinstalled + + +def validate_website_saas_directives(app): + def validator(enabled, app=app): + if enabled and isinstalled('orchestra.contrib.websites'): + from orchestra.contrib.websites import settings + if app not in settings.WEBSITES_SAAS_DIRECTIVES: + raise ValidationError(_("Allow custom URL is enabled for '%s', " + "but has no associated WEBSITES_SAAS_DIRECTIVES" % app)) + return validator diff --git a/orchestra/contrib/services/__init__.py b/orchestra/contrib/services/__init__.py new file mode 100644 index 00000000..cafed625 --- /dev/null +++ b/orchestra/contrib/services/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.services.apps.ServicesConfig' diff --git a/orchestra/contrib/services/actions.py b/orchestra/contrib/services/actions.py new file mode 100644 index 00000000..4e48b2d8 --- /dev/null +++ b/orchestra/contrib/services/actions.py @@ -0,0 +1,84 @@ +from django.contrib.admin import helpers +from django.urls import reverse +from django.db import transaction +from django.shortcuts import render, redirect +from django.template.response import TemplateResponse +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin.utils import get_object_from_url + + +@transaction.atomic +def update_orders(modeladmin, request, queryset, extra_context=None): + if not queryset: + return + if request.POST.get('post') == 'confirmation': + num = 0 + services = [] + for service in queryset: + updates = service.update_orders() + num += len(updates) + services.append(str(service.pk)) + modeladmin.log_change(request, service, _("Orders updated")) + if num == 1: + url = reverse('admin:orders_order_change', args=(updates[0][0].pk,)) + msg = _('One related order has benn updated') % url + else: + url = reverse('admin:orders_order_changelist') + url += '?service__in=%s' % ','.join(services) + msg = _('%s related orders have been updated') % (url, num) + modeladmin.message_user(request, mark_safe(msg)) + return + updates = [] + for service in queryset: + updates += service.update_orders(commit=False) + opts = modeladmin.model._meta + context = { + 'title': _("Update orders will cause the following."), + 'action_name': 'Update orders', + 'action_value': 'update_orders', + 'updates': updates, + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'obj': get_object_from_url(modeladmin, request), + } + return render(request, 'admin/services/service/update_orders.html', context) +update_orders.url_name = 'update-orders' +update_orders.short_description = _("Update orders") + + +def view_help(modeladmin, request, queryset): + opts = modeladmin.model._meta + context = { + 'title': _("Need some help?"), + 'opts': opts, + 'queryset': queryset, + 'obj': queryset.get(), + 'action_name': _("help"), + 'app_label': opts.app_label, + } + return TemplateResponse(request, 'admin/services/service/help.html', context) +view_help.url_name = 'help' +view_help.tool_description = _("Help") + + +def clone(modeladmin, request, queryset): + service = queryset.get() + fields = modeladmin.get_fields(request) + query = [] + for field in fields: + model_field = type(service)._meta.get_field(field) + if model_field.rel: + value = getattr(service, field + '_id') + elif 'Boolean' in model_field.__class__.__name__: + value = 'True' if getattr(service, field) else '' + else: + value = getattr(service, field) + query.append('%s=%s' % (field, value)) + opts = service._meta + url = reverse('admin:%s_%s_add' % (opts.app_label, opts.model_name)) + url += '?%s' % '&'.join(query) + return redirect(url) diff --git a/orchestra/contrib/services/admin.py b/orchestra/contrib/services/admin.py new file mode 100644 index 00000000..2f4b835f --- /dev/null +++ b/orchestra/contrib/services/admin.py @@ -0,0 +1,107 @@ +from django import forms +from django.urls import re_path as url +from django.contrib import admin +from django.urls import reverse +from django.template.response import TemplateResponse +from django.utils import timezone +from django.utils.html import format_html +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ChangeViewActionsMixin +from orchestra.admin.actions import disable, enable +from orchestra.core import services + +from .actions import update_orders, view_help, clone +from .models import Service + + +class ServiceAdmin(ChangeViewActionsMixin, admin.ModelAdmin): + list_display = ( + 'description', 'content_type', 'handler_type', 'num_orders', 'is_active' + ) + list_filter = ( + 'is_active', 'handler_type', 'is_fee', + ('content_type', admin.RelatedOnlyFieldListFilter), + ) + fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('description', 'content_type', 'match', 'periodic_update', + 'handler_type', 'ignore_superusers', 'is_active') + }), + (_("Billing options"), { + 'classes': ('wide',), + 'fields': ('billing_period', 'billing_point', 'is_fee', 'order_description', + 'ignore_period') + }), + (_("Pricing options"), { + 'classes': ('wide',), + 'fields': ('metric', 'pricing_period', 'rate_algorithm', + 'on_cancel', 'payment_style', 'tax', 'nominal_price') + }), + ) + actions = (update_orders, clone, disable, enable) + change_view_actions = actions + (view_help,) + change_form_template = 'admin/services/service/change_form.html' + + def get_urls(self): + """Returns the additional urls for the change view links""" + urls = super(ServiceAdmin, self).get_urls() + admin_site = self.admin_site + opts = self.model._meta + return [ + url('^add/help/$', + admin_site.admin_view(self.help_view), + name='%s_%s_help' % (opts.app_label, opts.model_name) + ) + ] + urls + + def formfield_for_dbfield(self, db_field, **kwargs): + """ Improve performance of account field and filter by account """ + if db_field.name == 'content_type': + models = [model._meta.model_name for model in services.get()] + queryset = db_field.remote_field.model.objects + kwargs['queryset'] = queryset.filter(model__in=models) + if db_field.name in ['match', 'metric', 'order_description']: + kwargs['widget'] = forms.TextInput(attrs={'size':'160'}) + return super(ServiceAdmin, self).formfield_for_dbfield(db_field, **kwargs) + + def num_orders(self, service): + num = service.orders__count + url = reverse('admin:orders_order_changelist') + url += '?service__id__exact=%i&is_active=True' % service.pk + return format_html('{}', url, num) + num_orders.short_description = _("Orders") + num_orders.admin_order_field = 'orders__count' + + def get_queryset(self, request): + qs = super(ServiceAdmin, self).get_queryset(request) + # Count active orders + qs = qs.extra(select={ + 'orders__count': ( + "SELECT COUNT(*) " + "FROM orders_order " + "WHERE orders_order.service_id = services_service.id AND (" + " orders_order.cancelled_on IS NULL OR" + " orders_order.cancelled_on > '%s' " + ")" % timezone.now() + ) + }) + return qs + + def help_view(self, request, *args): + opts = self.model._meta + context = { + 'add': True, + 'title': _("Need some help?"), + 'opts': opts, + 'obj': args[0].get() if args else None, + 'action_name': _("help"), + 'app_label': opts.app_label, + } + return TemplateResponse(request, 'admin/services/service/help.html', context) + help_view.url_name = 'help' + help_view.verbose_name = _("Help") + + +admin.site.register(Service, ServiceAdmin) diff --git a/orchestra/contrib/services/apps.py b/orchestra/contrib/services/apps.py new file mode 100644 index 00000000..0b6d76e9 --- /dev/null +++ b/orchestra/contrib/services/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + +from orchestra.core import administration +from orchestra.core.translations import ModelTranslation + + +class ServicesConfig(AppConfig): + name = 'orchestra.contrib.services' + verbose_name = 'Services' + + def ready(self): + from .models import Service + administration.register(Service, icon='price.png') + ModelTranslation.register(Service, ('description',)) diff --git a/orchestra/contrib/services/handlers.py b/orchestra/contrib/services/handlers.py new file mode 100644 index 00000000..64ceaf84 --- /dev/null +++ b/orchestra/contrib/services/handlers.py @@ -0,0 +1,662 @@ +import calendar +import datetime +import decimal +import math +from functools import cmp_to_key + +from dateutil import relativedelta +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.utils import timezone, translation +from django.utils.translation import gettext, gettext_lazy as _ + +from orchestra import plugins +from orchestra.utils.humanize import text2int +from orchestra.utils.python import AttrDict, format_exception + +from . import settings, helpers + + +class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): + """ + Separates all the logic of billing handling from the model allowing to better + customize its behaviout + + Relax and enjoy the journey. + """ + _PLAN = 'plan' + _COMPENSATION = 'compensation' + _PREPAY = 'prepay' + + model = None + + def __init__(self, service): + self.service = service + + def __getattr__(self, attr): + if attr.startswith('__'): + raise AttributeError(f'{self} does not have attribute {attr}') + return getattr(self.service, attr) + + @classmethod + def get_choices(cls): + choices = super().get_choices() + return [('', _("Default"))] + choices + + def validate_content_type(self, service): + pass + + def validate_expression(self, service, method): + try: + obj = service.content_type.model_class().objects.all()[0] + except IndexError: + return + try: + bool(getattr(self, method)(obj)) + except Exception as exc: + raise ValidationError(format_exception(exc)) + + def validate_match(self, service): + if not service.match: + service.match = 'True' + self.validate_expression(service, 'matches') + + def validate_metric(self, service): + self.validate_expression(service, 'get_metric') + + def validate_order_description(self, service): + self.validate_expression(service, 'get_order_description') + + def get_content_type(self): + if not self.model: + return self.content_type + app_label, model = self.model.split('.') + return ContentType.objects.get_by_natural_key(app_label, model.lower()) + + def get_expression_context(self, instance): + return { + 'instance': instance, + 'obj': instance, + 'gettext': gettext, + 'handler': self, + 'service': self.service, + instance._meta.model_name: instance, + 'math': math, + 'logsteps': lambda n, size=1: \ + round(n/(decimal.Decimal(size*10**int(math.log10(max(n, 1))))))*size*10**int(math.log10(max(n, 1))), + 'log10': math.log10, + 'Decimal': decimal.Decimal, + } + + def matches(self, instance): + if not self.match: + # Blank expressions always evaluate True + return True + safe_locals = self.get_expression_context(instance) + return eval(self.match, safe_locals) + + def get_ignore_delta(self): + if self.ignore_period == self.NEVER: + return None + value, unit = self.ignore_period.split('_') + value = text2int(value) + if unit.lower().startswith('day'): + return datetime.timedelta(days=value) + if unit.lower().startswith('month'): + return datetime.timedelta(months=value) + else: + raise ValueError("Unknown unit %s" % unit) + + def get_order_ignore(self, order): + """ service trial delta """ + ignore_delta = self.get_ignore_delta() + if ignore_delta and (order.cancelled_on-ignore_delta).date() <= order.registered_on: + return True + return order.ignore + + def get_ignore(self, instance): + if self.ignore_superusers: + account = getattr(instance, 'account', instance) + if account.type in settings.SERVICES_IGNORE_ACCOUNT_TYPE: + return True + if 'superuser' in settings.SERVICES_IGNORE_ACCOUNT_TYPE and account.is_superuser: + return True + return False + + def get_metric(self, instance): + if self.metric: + safe_locals = self.get_expression_context(instance) + try: + return eval(self.metric, safe_locals) + except Exception as exc: + raise type(exc)("'%s' evaluating metric for '%s' service" % (exc, self.service)) + + def get_order_description(self, instance): + safe_locals = self.get_expression_context(instance) + account = getattr(instance, 'account', instance) + with translation.override(account.language): + if not self.order_description: + return '%s: %s' % (gettext(self.description), instance) + return eval(self.order_description, safe_locals) + + def get_billing_point(self, order, bp=None, **options): + cachable = bool(self.billing_point == self.FIXED_DATE and not options.get('fixed_point')) + if not cachable or bp is None: + bp = options.get('billing_point') or timezone.now().date() + if not options.get('fixed_point'): + msg = ("Support for '%s' period and '%s' point is not implemented" + % (self.get_billing_period_display(), self.get_billing_point_display())) + if self.billing_period == self.MONTHLY: + date = bp + if self.payment_style == self.PREPAY: + date += relativedelta.relativedelta(months=1) + else: + date = timezone.now().date() + if self.billing_point == self.ON_REGISTER: + # handle edge cases of last day of the month: + # e.g. on March is 31 but on April 30 + last_day_of_month = calendar.monthrange(date.year, date.month)[1] + day = min(last_day_of_month, order.registered_on.day) + elif self.billing_point == self.FIXED_DATE: + day = 1 + else: + raise NotImplementedError(msg) + bp = datetime.date(year=date.year, month=date.month, day=day) + elif self.billing_period == self.ANUAL: + if self.billing_point == self.ON_REGISTER: + month = order.registered_on.month + day = order.registered_on.day + elif self.billing_point == self.FIXED_DATE: + month = settings.SERVICES_SERVICE_ANUAL_BILLING_MONTH + day = 1 + else: + raise NotImplementedError(msg) + year = bp.year + if self.payment_style == self.POSTPAY: + year = bp.year - relativedelta.relativedelta(years=1) + if bp.month >= month: + year = bp.year + 1 + + # handle edge cases of last day of the month: + # e.g. on March is 31 but on April 30 + last_day_of_month = calendar.monthrange(year,month)[1] + day = min(last_day_of_month, day) + bp = datetime.date(year=year, month=month, day=day) + elif self.billing_period == self.NEVER: + bp = order.registered_on + else: + raise NotImplementedError(msg) + if self.on_cancel != self.NOTHING and order.cancelled_on and order.cancelled_on < bp: + bp = order.cancelled_on + return bp + +# def aligned(self, date): +# if self.granularity == self.DAILY: +# return date +# elif self.granularity == self.MONTHLY: +# return datetime.date(year=date.year, month=date.month, day=1) +# elif self.granularity == self.ANUAL: +# return datetime.date(year=date.year, month=1, day=1) +# raise NotImplementedError + + def get_price_size(self, ini, end): + rdelta = relativedelta.relativedelta(end, ini) + anual_prepay_of_monthly_pricing = bool( + self.billing_period == self.ANUAL and + self.payment_style == self.PREPAY and + self.get_pricing_period() == self.MONTHLY) + if self.billing_period == self.MONTHLY or anual_prepay_of_monthly_pricing: + size = rdelta.years * 12 + size += rdelta.months + days = calendar.monthrange(end.year, end.month)[1] + size += decimal.Decimal(str(rdelta.days))/days + elif self.billing_period == self.ANUAL: + size = rdelta.years + size += decimal.Decimal(str(rdelta.months))/12 + days = 366 if calendar.isleap(end.year) else 365 + size += decimal.Decimal(str(rdelta.days))/days + elif self.billing_period == self.NEVER: + size = 1 + else: + raise NotImplementedError + size = round(size, 2) + return decimal.Decimal(str(size)) + + def get_pricing_slots(self, ini, end): + day = 1 + month = settings.SERVICES_SERVICE_ANUAL_BILLING_MONTH + if self.billing_point == self.ON_REGISTER: + day = ini.day + month = ini.month + period = self.get_pricing_period() + rdelta = self.get_pricing_rdelta() + if period == self.MONTHLY: + ini = datetime.date(year=ini.year, month=ini.month, day=day) + elif period == self.ANUAL: + ini = datetime.date(year=ini.year, month=month, day=day) + elif period == self.NEVER: + yield ini, end + raise StopIteration + else: + raise NotImplementedError + while True: + next = ini + rdelta + yield ini, next + if next >= end: + break + ini = next + + def get_pricing_rdelta(self): + period = self.get_pricing_period() + if period == self.MONTHLY: + return relativedelta.relativedelta(months=1) + elif period == self.ANUAL: + return relativedelta.relativedelta(years=1) + elif period == self.NEVER: + return None + + def generate_discount(self, line, dtype, price): + line.discounts.append(AttrDict(**{ + 'type': dtype, + 'total': price, + })) + + def generate_line(self, order, price, *dates, metric=1, discounts=None, computed=False): + """ + discounts: extra discounts to apply + computed: price = price*size already performed + """ + if len(dates) == 2: + ini, end = dates + elif len(dates) == 1: + ini, end = dates[0], dates[0] + else: + raise AttributeError("WTF is '%s'?" % dates) + discounts = discounts or () + + size = self.get_price_size(ini, end) + if not computed: + price = price * size + subtotal = self.nominal_price * size * metric + line = AttrDict(**{ + 'order': order, + 'subtotal': subtotal, + 'ini': ini, + 'end': end, + 'size': size, + 'metric': metric, + 'discounts': [], + }) + + if subtotal > price: + plan_discount = price-subtotal + self.generate_discount(line, self._PLAN, plan_discount) + subtotal += plan_discount + for dtype, dprice in discounts: + subtotal += dprice + # Prevent compensations/prepays to refund money + if dtype in (self._COMPENSATION, self._PREPAY) and subtotal < 0: + dprice -= subtotal + if dprice: + self.generate_discount(line, dtype, dprice) + return line + + def assign_compensations(self, givers, receivers, **options): + compensations = [] + for order in givers: + if order.billed_until and order.cancelled_on and order.cancelled_on < order.billed_until: + interval = helpers.Interval(order.cancelled_on, order.billed_until, order) + compensations.append(interval) + for order in receivers: + if not order.billed_until or order.billed_until < order.new_billed_until: + # receiver + ini = order.billed_until or order.registered_on + end = order.cancelled_on or datetime.date.max + interval = helpers.Interval(ini, end) + compensations, used_compensations = helpers.compensate(interval, compensations) + order._compensations = used_compensations + for comp in used_compensations: + comp.order.new_billed_until = min(comp.order.billed_until, comp.ini, + getattr(comp.order, 'new_billed_until', datetime.date.max)) + if options.get('commit', True): + for order in givers: + if hasattr(order, 'new_billed_until'): + order.billed_until = order.new_billed_until + order.save(update_fields=['billed_until']) + + def apply_compensations(self, order, only_beyond=False): + dsize = 0 + ini = order.billed_until or order.registered_on + end = order.new_billed_until + beyond = end + cend = None + new_end = None + for comp in getattr(order, '_compensations', []): + intersect = comp.intersect(helpers.Interval(ini=ini, end=end)) + if intersect: + cini, cend = intersect.ini, intersect.end + if comp.end > beyond: + cend = comp.end + new_end = cend + if only_beyond: + cini = beyond + elif only_beyond: + continue + dsize += self.get_price_size(cini, cend) + # Extend billing point a little bit to benefit from a substantial discount + elif comp.end > beyond and (comp.end-comp.ini).days > 3*(comp.ini-beyond).days: + cend = comp.end + new_end = cend + dsize += self.get_price_size(comp.ini, cend) + return dsize, new_end + + def get_register_or_renew_events(self, porders, ini, end): + counter = 0 + for order in porders: + bu = getattr(order, 'new_billed_until', order.billed_until) + if bu: + registered = order.registered_on + if registered > ini and registered <= end: + counter += 1 + if registered != bu and bu > ini and bu <= end: + counter += 1 + if order.billed_until and order.billed_until != bu: + if registered != order.billed_until and order.billed_until > ini and order.billed_until <= end: + counter += 1 + return counter + + def bill_concurrent_orders(self, account, porders, rates, ini, end): + # Concurrent + # Get pricing orders + priced = {} + for ini, end, orders in helpers.get_chunks(porders, ini, end): + size = self.get_price_size(ini, end) + metric = len(orders) + interval = helpers.Interval(ini=ini, end=end) + for position, order in enumerate(orders, start=1): + csize = 0 + compensations = getattr(order, '_compensations', []) + # Compensations < new_billed_until + for comp in compensations: + intersect = comp.intersect(interval) + if intersect: + csize += self.get_price_size(intersect.ini, intersect.end) + price = self.get_price(account, metric, position=position, rates=rates) + cprice = price * csize + price = price * size + if order in priced: + priced[order][0] += price + priced[order][1] += cprice + else: + priced[order] = [price, cprice] + lines = [] + for order, prices in priced.items(): + if hasattr(order, 'new_billed_until'): + discounts = () + # Generate lines and discounts from order.nominal_price + price, cprice = prices + a = order.id + # Compensations > new_billed_until + dsize, new_end = self.apply_compensations(order, only_beyond=True) + cprice += dsize*price + if cprice: + discounts = ( + (self._COMPENSATION, -cprice), + ) + if new_end: + size = self.get_price_size(order.new_billed_until, new_end) + price += price*size + order.new_billed_until = new_end + ini = order.billed_until or order.registered_on + end = new_end or order.new_billed_until + line = self.generate_line( + order, price, ini, end, discounts=discounts, computed=True) + lines.append(line) + return lines + + def bill_registered_or_renew_events(self, account, porders, rates): + # Before registration + lines = [] + rdelta = self.get_pricing_rdelta() + if not rdelta: + raise NotImplementedError + for position, order in enumerate(porders, start=1): + if hasattr(order, 'new_billed_until'): + pend = order.billed_until or order.registered_on + pini = pend - rdelta + metric = self.get_register_or_renew_events(porders, pini, pend) + position = min(position, metric) + price = self.get_price(account, metric, position=position, rates=rates) + ini = order.billed_until or order.registered_on + end = order.new_billed_until + discounts = () + dsize, new_end = self.apply_compensations(order) + if dsize: + discounts=( + (self._COMPENSATION, -dsize*price), + ) + if new_end: + order.new_billed_until = new_end + end = new_end + line = self.generate_line(order, price, ini, end, discounts=discounts) + lines.append(line) + return lines + + def bill_with_orders(self, orders, account, **options): + # For the "boundary conditions" just think that: + # date(2011, 1, 1) is equivalent to datetime(2011, 1, 1, 0, 0, 0) + # In most cases: + # ini >= registered_date, end < registered_date + # boundary lookup and exclude cancelled and billed + orders_ = [] + bp = None + ini = datetime.date.max + end = datetime.date.min + for order in orders: + cini = order.registered_on + if order.billed_until: + # exclude cancelled and billed + if self.on_cancel != self.REFUND: + if order.cancelled_on and order.billed_until > order.cancelled_on: + continue + cini = order.billed_until + bp = self.get_billing_point(order, bp=bp, **options) + if order.billed_until and order.billed_until >= bp: + continue + order.new_billed_until = bp + ini = min(ini, cini) + end = max(end, bp) + orders_.append(order) + orders = orders_ + + # Compensation + related_orders = account.orders.filter(service=self.service) + if self.payment_style == self.PREPAY and self.on_cancel == self.COMPENSATE: + # Get orders pending for compensation + givers = list(related_orders.givers(ini, end)) + givers = sorted(givers, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on)) + orders = sorted(orders, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on)) + self.assign_compensations(givers, orders, **options) + rates = self.get_rates(account) + has_billing_period = self.billing_period != self.NEVER + has_pricing_period = self.get_pricing_period() != self.NEVER + if rates and (has_billing_period or has_pricing_period): + concurrent = has_billing_period and not has_pricing_period + if not concurrent: + rdelta = self.get_pricing_rdelta() + ini -= rdelta + porders = related_orders.pricing_orders(ini, end) + porders = list(set(orders).union(set(porders))) + porders = sorted(porders, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on)) + if concurrent: + # Periodic billing with no pricing period + lines = self.bill_concurrent_orders(account, porders, rates, ini, end) + else: + # Periodic and one-time billing with pricing period + lines = self.bill_registered_or_renew_events(account, porders, rates) + else: + # No rates optimization or one-time billing without pricing period + lines = [] + price = self.nominal_price + # Calculate nominal price + for order in orders: + ini = order.billed_until or order.registered_on + end = order.new_billed_until + discounts = () + dsize, new_end = self.apply_compensations(order) + if dsize: + discounts=( + (self._COMPENSATION, -dsize*price), + ) + if new_end: + order.new_billed_until = new_end + end = new_end + line = self.generate_line(order, price, ini, end, discounts=discounts) + lines.append(line) + return lines + + def bill_with_metric(self, orders, account, **options): + lines = [] + bp = None + for order in orders: + prepay_discount = 0 + bp = self.get_billing_point(order, bp=bp, **options) + recharged_until = datetime.date.min + + if (self.billing_period != self.NEVER and + self.get_pricing_period() == self.NEVER and + self.payment_style == self.PREPAY and order.billed_on): + # Recharge + if self.payment_style == self.PREPAY and order.billed_on: + recharges = [] + rini = order.billed_on + rend = min(bp, order.billed_until) + bmetric = order.billed_metric + if bmetric is None: + bmetric = order.get_metric(order.billed_on) + bsize = self.get_price_size(rini, rend) + prepay_discount = self.get_price(account, bmetric) * bsize + prepay_discount = round(prepay_discount, 2) + for cini, cend, metric in order.get_metric(rini, rend, changes=True): + cini = max(cini, rini) + size = self.get_price_size(cini, cend) + price = self.get_price(account, metric) * size + discounts = () + discount = min(price, max(prepay_discount, 0)) + prepay_discount -= price + if discount > 0: + price -= discount + discounts = ( + (self._PREPAY, -discount), + ) + # Don't overdload bills with lots of lines + if price > 0: + recharges.append((order, price, cini, cend, metric, discounts)) + if prepay_discount < 0: + # User has prepaid less than the actual consumption + for order, price, cini, cend, metric, discounts in recharges: + if discounts: + price -= discounts[0][1] + line = self.generate_line(order, price, cini, cend, metric=metric, + computed=True, discounts=discounts) + lines.append(line) + recharged_until = cend + if order.billed_until and order.cancelled_on and order.cancelled_on >= order.billed_until: + # Cancelled order + continue + if self.billing_period != self.NEVER: + ini = order.billed_until or order.registered_on +# ini = max(order.billed_until or order.registered_on, recharged_until) + # Periodic billing + if bp <= ini: + # Already billed + continue + order.new_billed_until = bp + if self.get_pricing_period() == self.NEVER: + # Changes (Mailbox disk-like) + for cini, cend, metric in order.get_metric(ini, bp, changes=True): + cini = max(recharged_until, cini) + price = self.get_price(account, metric) + discounts = () + # Since the current datamodel can't guarantee to retrieve the exact + # state for calculating prepay_discount (service price could have change) + # maybe is it better not to discount anything. +# discount = min(price, max(prepay_discount, 0)) +# if discount > 0: +# price -= discount +# prepay_discount -= discount +# discounts = ( +# (self._PREPAY, -discount), +# ) + if metric > 0: + line = self.generate_line(order, price, cini, cend, metric=metric, + discounts=discounts) + lines.append(line) + elif self.get_pricing_period() == self.billing_period: + # pricing_slots (Traffic-like) + if self.payment_style == self.PREPAY: + raise NotImplementedError( + "Metric with prepay and pricing_period == billing_period") + for cini, cend in self.get_pricing_slots(ini, bp): + metric = order.get_metric(cini, cend) + price = self.get_price(account, metric) + discounts = () +# discount = min(price, max(prepay_discount, 0)) +# if discount > 0: +# price -= discount +# prepay_discount -= discount +# discounts = ( +# (self._PREPAY, -discount), +# ) + if metric > 0: + line = self.generate_line(order, price, cini, cend, metric=metric, + discounts=discounts) + lines.append(line) + elif self.get_pricing_period() in (self.MONTHLY, self.ANUAL): + if self.payment_style == self.PREPAY: + # Traffic Prepay + metric = order.get_metric(timezone.now().date()) + if metric > 0: + price = self.get_price(account, metric) + for cini, cend in self.get_pricing_slots(ini, bp): + line = self.generate_line(order, price, cini, cend, metric=metric) + lines.append(line) + else: + raise NotImplementedError( + "Metric with postpay and pricing_period in (monthly, anual)") + else: + raise NotImplementedError + else: + # One-time billing + if order.billed_until: + continue + date = timezone.now().date() + order.new_billed_until = date + if self.get_pricing_period() == self.NEVER: + # get metric (Job-like) + metric = order.get_metric(date) + price = self.get_price(account, metric) + line = self.generate_line(order, price, date, metric=metric) + lines.append(line) + else: + raise NotImplementedError + # Last processed metric for futrue recharges + order.new_billed_metric = metric + return lines + + def generate_bill_lines(self, orders, account, **options): + if options.get('proforma', False): + options['commit'] = False + if not self.metric: + lines = self.bill_with_orders(orders, account, **options) + else: + lines = self.bill_with_metric(orders, account, **options) + if options.get('commit', True): + now = timezone.now().date() + for line in lines: + order = line.order + order.billed_on = now + order.billed_metric = getattr(order, 'new_billed_metric', order.billed_metric) + order.billed_until = getattr(order, 'new_billed_until', order.billed_until) + order.save(update_fields=('billed_on', 'billed_until', 'billed_metric')) + return lines diff --git a/orchestra/contrib/services/helpers.py b/orchestra/contrib/services/helpers.py new file mode 100644 index 00000000..68046597 --- /dev/null +++ b/orchestra/contrib/services/helpers.py @@ -0,0 +1,149 @@ +from django.utils.text import format_lazy +from django.utils.translation import gettext_lazy + + +def get_chunks(porders, ini, end, ix=0): + if ix >= len(porders): + return [[ini, end, []]] + order = porders[ix] + ix += 1 + bu = getattr(order, 'new_billed_until', order.billed_until) + if not bu or bu <= ini or order.registered_on >= end: + return get_chunks(porders, ini, end, ix=ix) + result = [] + if order.registered_on < end and order.registered_on > ini: + ro = order.registered_on + result = get_chunks(porders, ini, ro, ix=ix) + ini = ro + if bu < end: + result += get_chunks(porders, bu, end, ix=ix) + end = bu + chunks = get_chunks(porders, ini, end, ix=ix) + for chunk in chunks: + chunk[2].insert(0, order) + result.append(chunk) + return result + + +def cmp_billed_until_or_registered_on(a, b): + """ + 1) billed_until greater first + 2) registered_on smaller first + """ + if a.billed_until == b.billed_until: + # Use pk which is more reliable than registered_on date + return a.id-b.id + elif a.billed_until and b.billed_until: + return (b.billed_until-a.billed_until).days + elif a.billed_until: + return (b.registered_on-a.billed_until).days + return (b.billed_until-a.registered_on).days + + +class Interval(object): + def __init__(self, ini, end, order=None): + self.ini = ini + self.end = end + self.order = order + + def __len__(self): + return max((self.end-self.ini).days, 0) + + def __sub__(self, other): + remaining = [] + if self.ini < other.ini: + remaining.append(Interval(self.ini, min(self.end, other.ini), self.order)) + if self.end > other.end: + remaining.append(Interval(max(self.ini,other.end), self.end, self.order)) + return remaining + + def __repr__(self): + return "".format( + ini=self.ini.strftime('%Y-%-m-%-d'), + end=self.end.strftime('%Y-%-m-%-d') + ) + + def intersect(self, other, remaining_self=None, remaining_other=None): + if remaining_self is not None: + remaining_self += (self - other) + if remaining_other is not None: + remaining_other += (other - self) + result = Interval(max(self.ini, other.ini), min(self.end, other.end), self.order) + if len(result)>0: + return result + else: + return None + + def intersect_set(self, others, remaining_self=None, remaining_other=None): + intersections = [] + for interval in others: + intersection = self.intersect(interval, remaining_self, remaining_other) + if intersection: + intersections.append(intersection) + return intersections + + +def get_intersections(order_intervals, compensations): + intersections = [] + for compensation in compensations: + intersection = compensation.intersect_set(order_intervals) + length = 0 + for intersection_interval in intersection: + length += len(intersection_interval) + intersections.append((length, compensation)) + return sorted(intersections, key=lambda i: i[0]) + + +def intersect(compensation, order_intervals): + # Intervals should not overlap + compensated = [] + not_compensated = [] + unused_compensation = [] + for interval in order_intervals: + compensated.append(compensation.intersect(interval, unused_compensation, not_compensated)) + return (compensated, not_compensated, unused_compensation) + + +def apply_compensation(order, compensation): + remaining_order = [] + remaining_compensation = [] + applied_compensation = compensation.intersect_set(order, remaining_compensation, remaining_order) + return applied_compensation, remaining_order, remaining_compensation + + +def update_intersections(not_compensated, compensations): + # TODO can be optimized + compensation_intervals = [] + for __, compensation in compensations: + compensation_intervals.append(compensation) + return get_intersections(not_compensated, compensation_intervals) + + +def compensate(order, compensations): + remaining_interval = [order] + ordered_intersections = get_intersections(remaining_interval, compensations) + applied_compensations = [] + remaining_compensations = [] + while ordered_intersections and ordered_intersections[len(ordered_intersections)-1][0]>0: + # Apply the first compensation: + __, compensation = ordered_intersections.pop() + (applied_compensation, remaining_interval, remaining_compensation) = apply_compensation(remaining_interval, compensation) + remaining_compensations += remaining_compensation + applied_compensations += applied_compensation + ordered_intersections = update_intersections(remaining_interval, ordered_intersections) + for __, compensation in ordered_intersections: + remaining_compensations.append(compensation) + return remaining_compensations, applied_compensations + + +def get_rate_methods_help_text(rate_class): + method_help_texts = [ + format_lazy('{}' * 4, *['
      ', method.verbose_name, ': ', method.help_text]) + for method in rate_class.get_methods().values() + ] + prefix = gettext_lazy("Algorithm used to interprete the rating table.") + help_text_items = [prefix] + method_help_texts + return format_lazy( + '{}' * len(help_text_items), + *help_text_items + ) diff --git a/orchestra/contrib/services/models.py b/orchestra/contrib/services/models.py new file mode 100644 index 00000000..0dc8bb80 --- /dev/null +++ b/orchestra/contrib/services/models.py @@ -0,0 +1,266 @@ +import calendar +import decimal +from orchestra.contrib.services import helpers + +from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.apps import apps +from django.utils.functional import cached_property +from django.utils.module_loading import autodiscover_modules +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import caches, validators +from orchestra.utils.python import import_class + +from . import settings +from .handlers import ServiceHandler +from .helpers import get_rate_methods_help_text + + +autodiscover_modules('handlers') + +rate_class = import_class(settings.SERVICES_RATE_CLASS) + + +class ServiceQuerySet(models.QuerySet): + def filter_by_instance(self, instance): + cache = caches.get_request_cache() + ct = ContentType.objects.get_for_model(instance) + key = 'services.Service-%i' % ct.pk + services = cache.get(key) + if services is None: + services = self.filter(content_type=ct, is_active=True) + cache.set(key, services) + return services + + +class Service(models.Model): + NEVER = '' +# DAILY = 'DAILY' + MONTHLY = 'MONTHLY' + ANUAL = 'ANUAL' + ONE_DAY = 'ONE_DAY' + TWO_DAYS = 'TWO_DAYS' + TEN_DAYS = 'TEN_DAYS' + ONE_MONTH = 'ONE_MONTH' + ALWAYS = 'ALWAYS' + ON_REGISTER = 'ON_REGISTER' + FIXED_DATE = 'ON_FIXED_DATE' + BILLING_PERIOD = 'BILLING_PERIOD' + REGISTER_OR_RENEW = 'REGISTER_OR_RENEW' + CONCURRENT = 'CONCURRENT' + NOTHING = 'NOTHING' + DISCOUNT = 'DISCOUNT' + COMPENSATE = 'COMPENSATE' + REFUND = 'REFUND' + PREPAY = 'PREPAY' + POSTPAY = 'POSTPAY' + + _ignore_types = ' and '.join( + ', '.join(settings.SERVICES_IGNORE_ACCOUNT_TYPE).rsplit(', ', 1)).lower() + + description = models.CharField(_("description"), max_length=256, unique=True) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, + verbose_name=_("content type"), + help_text=_("Content type of the related service objects.")) + match = models.CharField(_("match"), max_length=256, blank=True, + help_text=_( + "Python expression " + "that designates wheter a content_type object is related to this service " + "or not, always evaluates True when left blank. " + "Related instance can be instantiated with instance keyword or " + "content_type.model_name.
    " + " databaseuser.type == 'MYSQL'
    " + " miscellaneous.active and str(miscellaneous.identifier).endswith(('.org', '.net', '.com'))
    " + " contractedplan.plan.name == 'association_fee''
    " + " instance.active")) + periodic_update = models.BooleanField(_("periodic update"), default=False, + help_text=_("Whether a periodic update of this service orders should be performed or not. " + "Needed for match definitions that depend on complex model interactions, " + "where content type model save and delete operations are not enought.")) + handler_type = models.CharField(_("handler"), max_length=256, blank=True, + help_text=_("Handler used for processing this Service. A handler enables customized " + "behaviour far beyond what options here allow to."), + choices=ServiceHandler.get_choices()) + is_active = models.BooleanField(_("active"), default=True) + ignore_superusers = models.BooleanField(_("ignore %s") % _ignore_types, default=True, + help_text=_("Designates whether %s orders are marked as ignored by default or not.") % _ignore_types) + # Billing + billing_period = models.CharField(_("billing period"), max_length=16, + help_text=_("Renewal period for recurring invoicing."), + choices=( + (NEVER, _("One time service")), + (MONTHLY, _("Monthly billing")), + (ANUAL, _("Anual billing")), + ), + default=ANUAL, blank=True) + billing_point = models.CharField(_("billing point"), max_length=16, + help_text=_("Reference point for calculating the renewal date on recurring invoices"), + choices=( + (ON_REGISTER, _("Registration date")), + (FIXED_DATE, _("Every %(month)s") % { + 'month': calendar.month_name[settings.SERVICES_SERVICE_ANUAL_BILLING_MONTH] + }), + ), + default=FIXED_DATE) + is_fee = models.BooleanField(_("fee"), default=False, + help_text=_("Designates whether this service should be billed as membership fee or not")) + order_description = models.CharField(_("Order description"), max_length=256, blank=True, + help_text=_( + "Python expression " + "used for generating the description for the bill lines of this services.
    " + "Defaults to '%s: %s' % (gettext(handler.description), instance)" + )) + ignore_period = models.CharField(_("ignore period"), max_length=16, blank=True, + help_text=_("Period in which orders will be ignored if cancelled. " + "Useful for designating trial periods"), + choices=( + (NEVER, _("Never")), + (ONE_DAY, _("One day")), + (TWO_DAYS, _("Two days")), + (TEN_DAYS, _("Ten days")), + (ONE_MONTH, _("One month")), + ), + default=settings.SERVICES_DEFAULT_IGNORE_PERIOD) + # Pricing + metric = models.CharField(_("metric"), max_length=256, blank=True, + help_text=_( + "Python expression " + "used for obtinging the metric value for the pricing rate computation. " + "Number of orders is used when left blank. Related instance can be instantiated " + "with instance keyword or content_type.model_name.
    " + " max((mailbox.resources.disk.allocated or 0) -1, 0)
    " + " miscellaneous.amount
    " + " max((account.resources.traffic.used or 0) -" + " getattr(account.miscellaneous.filter(is_active=True," + " service__name='traffic-prepay').last(), 'amount', 0), 0)")) + nominal_price = models.DecimalField(_("nominal price"), max_digits=12, + decimal_places=2) + tax = models.PositiveIntegerField(_("tax"), choices=settings.SERVICES_SERVICE_TAXES, + default=settings.SERVICES_SERVICE_DEFAULT_TAX) + pricing_period = models.CharField(_("pricing period"), max_length=16, blank=True, + help_text=_("Time period that is used for computing the rate metric."), + choices=( + (NEVER, _("Current value")), + (BILLING_PERIOD, _("Same as billing period")), + (MONTHLY, _("Monthly data")), + (ANUAL, _("Anual data")), + ), + default=BILLING_PERIOD) + rate_algorithm = models.CharField( + _("rate algorithm"), max_length=64, + choices=rate_class.get_choices(), + default=rate_class.get_default(), + help_text=get_rate_methods_help_text(rate_class), + ) + on_cancel = models.CharField(_("on cancel"), max_length=16, + help_text=_("Defines the cancellation behaviour of this service."), + choices=( + (NOTHING, _("Nothing")), + (DISCOUNT, _("Discount")), + (COMPENSATE, _("Compensat")), + (REFUND, _("Refund")), + ), + default=DISCOUNT) + payment_style = models.CharField(_("payment style"), max_length=16, + help_text=_("Designates whether this service should be paid after " + "consumtion (postpay/on demand) or prepaid."), + choices=( + (PREPAY, _("Prepay")), + (POSTPAY, _("Postpay (on demand)")), + ), + default=PREPAY) + + objects = ServiceQuerySet.as_manager() + + def __str__(self): + return self.description + + @cached_property + def handler(self): + """ Accessor of this service handler instance """ + if self.handler_type: + return ServiceHandler.get(self.handler_type)(self) + return ServiceHandler(self) + + def clean(self): + self.description = self.description.strip() + if hasattr(self, 'content_type'): + validators.all_valid({ + 'content_type': (self.handler.validate_content_type, self), + 'match': (self.handler.validate_match, self), + 'metric': (self.handler.validate_metric, self), + 'order_description': (self.handler.validate_order_description, self), + }) + + def get_pricing_period(self): + if self.pricing_period == self.BILLING_PERIOD: + return self.billing_period + return self.pricing_period + + def get_price(self, account, metric, rates=None, position=None): + """ + if position is provided an specific price for that position is returned, + accumulated price is returned otherwise + """ + if rates is None: + rates = self.get_rates(account) + if rates: + rates = self.rate_method(rates, metric) + if not rates: + rates = [{ + 'quantity': metric, + 'price': self.nominal_price, + }] + counter = 0 + if position is None: + ant_counter = 0 + accumulated = 0 + for rate in rates: + counter += rate['quantity'] + if counter >= metric: + counter = metric + accumulated += (counter - ant_counter) * rate['price'] + accumulated = round(accumulated, 2) + return decimal.Decimal(str(accumulated)) + ant_counter = counter + accumulated += rate['price'] * rate['quantity'] + raise RuntimeError("Rating algorithm bad result") + else: + if metric < position: + raise ValueError("Metric can not be less than the position.") + for rate in rates: + counter += rate['quantity'] + if counter >= position: + price = round(rate['price'], 2) + return decimal.Decimal(str(rate['price'])) + raise RuntimeError("Rating algorithm bad result") + + def get_rates(self, account, cache=True): + # rates are cached per account + if not cache: + return self.rates.by_account(account) + if not hasattr(self, '__cached_rates'): + self.__cached_rates = {} + try: + return self.__cached_rates[account.id] + except KeyError: + rates = self.rates.by_account(account) + self.__cached_rates[account.id] = rates + return rates + + @property + def rate_method(self): + return rate_class.get_methods()[self.rate_algorithm] + + def update_orders(self, commit=True): + order_model = apps.get_model(settings.SERVICES_ORDER_MODEL) + manager = order_model.objects + related_model = self.content_type.model_class() + updates = [] + queryset = related_model.objects.all() + if related_model._meta.model_name != 'account': + queryset = queryset.select_related('account').all() + for instance in queryset: + updates += manager.update_by_instance(instance, service=self, commit=commit) + return updates diff --git a/orchestra/contrib/services/settings.py b/orchestra/contrib/services/settings.py new file mode 100644 index 00000000..5c783067 --- /dev/null +++ b/orchestra/contrib/services/settings.py @@ -0,0 +1,51 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting + + +SERVICES_SERVICE_TAXES = Setting('SERVICES_SERVICE_TAXES', + ( + (0, _("Duty free")), + (21, "21%"), + ), + validators=[Setting.validate_choices] +) + + +SERVICES_SERVICE_DEFAULT_TAX = Setting('SERVICES_SERVICE_DEFAULT_TAX', + 0, + choices=SERVICES_SERVICE_TAXES +) + + +SERVICES_SERVICE_ANUAL_BILLING_MONTH = Setting('SERVICES_SERVICE_ANUAL_BILLING_MONTH', + 1, + choices=tuple(enumerate( + ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'), 1)) +) + + +SERVICES_ORDER_MODEL = Setting('SERVICES_ORDER_MODEL', + 'orders.Order', + validators=[Setting.validate_model_label] +) + + +SERVICES_RATE_CLASS = Setting('SERVICES_RATE_CLASS', + 'orchestra.contrib.plans.models.Rate', + validators=[Setting.validate_import_class] +) + + +SERVICES_DEFAULT_IGNORE_PERIOD = Setting('SERVICES_DEFAULT_IGNORE_PERIOD', + 'TEN_DAYS' +) + + +SERVICES_IGNORE_ACCOUNT_TYPE = Setting('SERVICES_IGNORE_ACCOUNT_TYPE', + ( + 'superuser', + 'STAFF', + 'FRIEND', + ), +) diff --git a/orchestra/contrib/services/static/services/img/services.png b/orchestra/contrib/services/static/services/img/services.png new file mode 100644 index 00000000..0cf8792c Binary files /dev/null and b/orchestra/contrib/services/static/services/img/services.png differ diff --git a/orchestra/contrib/services/static/services/img/services.svg b/orchestra/contrib/services/static/services/img/services.svg new file mode 100644 index 00000000..0b40c712 --- /dev/null +++ b/orchestra/contrib/services/static/services/img/services.svg @@ -0,0 +1,485 @@ + + + + + + + + + + image/svg+xml + + + + + + + Orders + Metric + Periodicbilling + One-timeservice + Pricingperiod + No pricingperiod + Periodicbilling + One-timeservice + Pricingperiod + No pricingperiod + Pricingperiod + No pricingperiod + Pricingperiod + No pricingperiod + Mail accountsConcurrent (changes)Compensate on prepay + DomainsRegister or renew eventsCompensate on prepay + PlansAlways one order + CMS installationRegister or renew events + Traffic consumptionMetric period lookupPrepay and != billing_period NotImplemented + Mailbox sizeConcurrent (changes) + JobsLast known metric + NotImplement + + + + + + + + + + + + + + diff --git a/orchestra/contrib/services/tasks.py b/orchestra/contrib/services/tasks.py new file mode 100644 index 00000000..87bf7bbb --- /dev/null +++ b/orchestra/contrib/services/tasks.py @@ -0,0 +1,13 @@ +from celery.task.schedules import crontab + +from orchestra.contrib.tasks import periodic_task + +from .models import Service + + +@periodic_task(run_every=crontab(hour=5, minute=30)) +def update_service_orders(): + updates = [] + for service in Service.objects.filter(periodic_update=True): + updates += service.update_orders(commit=True) + return updates diff --git a/orchestra/contrib/services/templates/admin/services/service/change_form.html b/orchestra/contrib/services/templates/admin/services/service/change_form.html new file mode 100644 index 00000000..8a32497e --- /dev/null +++ b/orchestra/contrib/services/templates/admin/services/service/change_form.html @@ -0,0 +1,180 @@ +{% extends "orchestra/admin/change_form.html" %} +{% load i18n admin_urls static admin_modify utils %} + + +{% block object-tools %} +{% if add %} +
      +
      • +
    • + {% trans "Help" %} +
      • +
    +{% endif %} +{{ block.super }} +{% endblock %} diff --git a/orchestra/contrib/services/templates/admin/services/service/help.html b/orchestra/contrib/services/templates/admin/services/service/help.html new file mode 100644 index 00000000..27e29c48 --- /dev/null +++ b/orchestra/contrib/services/templates/admin/services/service/help.html @@ -0,0 +1,12 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} +{% load i18n l10n %} +{% load admin_urls static utils %} + + +{% block content %} +
    +
    + +
    +
    +{% endblock %} diff --git a/orchestra/contrib/services/templates/admin/services/service/update_orders.html b/orchestra/contrib/services/templates/admin/services/service/update_orders.html new file mode 100644 index 00000000..01882218 --- /dev/null +++ b/orchestra/contrib/services/templates/admin/services/service/update_orders.html @@ -0,0 +1,52 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} +{% load i18n l10n %} +{% load admin_urls static utils %} + + +{% block content %} +
    +
    +
    + + + + + + + + + + + + + + + + {% for order, action in updates %} + + + + + + + + + + + + {% endfor %} + +
    Action
    ID
    Service
    Account
    Content object
    Registered on
    Billed until
    Cancelled on
    Ignored
    {{ action.capitalize }}{% if order.pk %}{{ order.id }}{% endif %}{{ order.service }}{{ order.account }}{{ order.content_object }}{{ order.registered_on }}{{ order.billed_unitl }}{{ order.canncelled_on }}
    +
    +
    + +
    {% csrf_token %} + {% for obj in queryset %} + + {% endfor %} + + + +
    +{% endblock %} + diff --git a/orchestra/contrib/services/tests/__init__.py b/orchestra/contrib/services/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/services/tests/functional_tests/__init__.py b/orchestra/contrib/services/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/services/tests/functional_tests/test_domain.py b/orchestra/contrib/services/tests/functional_tests/test_domain.py new file mode 100644 index 00000000..a45c3ec2 --- /dev/null +++ b/orchestra/contrib/services/tests/functional_tests/test_domain.py @@ -0,0 +1,138 @@ +from django.contrib.contenttypes.models import ContentType + +from orchestra.contrib.miscellaneous.models import MiscService, Miscellaneous +from orchestra.contrib.plans.models import Plan +from orchestra.utils.tests import random_ascii, BaseTestCase + +from ...models import Service + + +class DomainBillingTest(BaseTestCase): + def create_domain_service(self): + service = Service.objects.create( + description="Domain .ES", + content_type=ContentType.objects.get_for_model(Miscellaneous), + match="miscellaneous.is_active and miscellaneous.service.name.lower() == 'domain .es'", + billing_period=Service.ANUAL, + billing_point=Service.ON_REGISTER, + is_fee=False, + metric='', + pricing_period=Service.BILLING_PERIOD, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.NOTHING, + payment_style=Service.PREPAY, + tax=0, + nominal_price=10 + ) + plan = Plan.objects.create(is_default=True, name='Default') + service.rates.create(plan=plan, quantity=0, price=0) + service.rates.create(plan=plan, quantity=2, price=10) + service.rates.create(plan=plan, quantity=4, price=9) + service.rates.create(plan=plan, quantity=6, price=6) + return service + + def create_domain(self, account=None): + if not account: + account = self.create_account() + domain_name = '%s.es' % random_ascii(10) + domain_service, __ = MiscService.objects.get_or_create(name='domain .es', + description='Domain .ES') + return account.miscellaneous.create(service=domain_service, description=domain_name) + + def test_domain(self): + self.create_domain_service() + account = self.create_account() + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(0, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(10, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(20, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(29, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(38, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(44, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(50, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill() + self.assertEqual(56, bills[0].total) + + def test_domain_proforma(self): + self.create_domain_service() + account = self.create_account() + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(0, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(10, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(20, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(29, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(38, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(44, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(50, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True, new_open=True) + self.assertEqual(56, bills[0].total) + + def test_domain_cumulative(self): + self.create_domain_service() + account = self.create_account() + self.create_domain(account=account) + bills = account.orders.bill(proforma=True) + self.assertEqual(0, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True) + self.assertEqual(10, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(proforma=True) + self.assertEqual(30, bills[0].total) + + def test_domain_new_open(self): + self.create_domain_service() + account = self.create_account() + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(0, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(10, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(10, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(9, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(9, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(6, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(6, bills[0].total) + self.create_domain(account=account) + bills = account.orders.bill(new_open=True) + self.assertEqual(6, bills[0].total) + diff --git a/orchestra/contrib/services/tests/functional_tests/test_ftp.py b/orchestra/contrib/services/tests/functional_tests/test_ftp.py new file mode 100644 index 00000000..03f51be6 --- /dev/null +++ b/orchestra/contrib/services/tests/functional_tests/test_ftp.py @@ -0,0 +1,102 @@ +import decimal +import datetime + +from dateutil.relativedelta import relativedelta +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone + +from orchestra.contrib.systemusers.models import SystemUser +from orchestra.utils.tests import random_ascii, BaseTestCase + +from ... import settings +from ...models import Service + + +class FTPBillingTest(BaseTestCase): + DEPENDENCIES = ( + 'orchestra.contrib.orders', + 'orchestra.contrib.plans', + 'orchestra.contrib.systemusers', + ) + + def create_ftp_service(self): + return Service.objects.create( + description="FTP Account", + content_type=ContentType.objects.get_for_model(SystemUser), + match='not systemuser.is_main', + billing_period=Service.ANUAL, + billing_point=Service.FIXED_DATE, + is_fee=False, + metric='', + pricing_period=Service.NEVER, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.COMPENSATE, + payment_style=Service.PREPAY, + tax=0, + nominal_price=10, + ) + + def create_ftp(self, account=None): + if not account: + account = self.create_account() + username = '%s_ftp' % random_ascii(10) + return SystemUser.objects.create_user(username, account=account) + + def test_ftp_account_1_year_fiexed(self): + service = self.create_ftp_service() + self.create_ftp() + self.assertEqual(1, service.orders.count()) + bp = timezone.now().date() + relativedelta(years=1) + bills = service.orders.bill(billing_point=bp, fixed_point=True) + self.assertEqual(10, bills[0].total) + + def test_ftp_account_2_year_fiexed(self): + service = self.create_ftp_service() + self.create_ftp() + bp = timezone.now().date() + relativedelta(years=2) + bills = service.orders.bill(billing_point=bp, fixed_point=True) + self.assertEqual(20, bills[0].total) + + def test_ftp_account_6_month_fixed(self): + service = self.create_ftp_service() + self.create_ftp() + bp = timezone.now().date() + relativedelta(months=6) + bills = service.orders.bill(billing_point=bp, fixed_point=True) + self.assertEqual(5, bills[0].total) + + def test_ftp_account_next_billing_point(self): + service = self.create_ftp_service() + self.create_ftp() + now = timezone.now().date() + bp_month = settings.SERVICES_SERVICE_ANUAL_BILLING_MONTH + if now.month > bp_month: + bp = datetime.date(year=now.year+1, month=bp_month, day=1) + else: + bp = datetime.date(year=now.year, month=bp_month, day=1) + bills = service.orders.bill(billing_point=now, fixed_point=False) + size = decimal.Decimal((bp - now).days)/365 + error = decimal.Decimal(0.05) + self.assertGreater(10*size+error*(10*size), bills[0].total) + self.assertLess(10*size-error*(10*size), bills[0].total) + + def test_ftp_account_with_compensation(self): + account = self.create_account() + self.create_ftp_service() + user = self.create_ftp(account=account) + first_bp = timezone.now().date() + relativedelta(years=2) + bills = account.orders.bill(billing_point=first_bp, fixed_point=True) + self.assertEqual(1, account.orders.active().count()) + user.delete() + self.assertEqual(0, account.orders.active().count()) + user = self.create_ftp(account=account) + self.assertEqual(1, account.orders.active().count()) + self.assertEqual(2, account.orders.count()) + bp = timezone.now().date() + relativedelta(years=1) + bills = account.orders.bill(billing_point=bp, fixed_point=True, new_open=True) + discount = bills[0].lines.order_by('id')[0].sublines.get() + self.assertEqual(decimal.Decimal(-20), discount.total) + order = account.orders.order_by('id').first() + self.assertEqual(order.cancelled_on, order.billed_until) + order = account.orders.order_by('-id').first() + self.assertEqual(first_bp, order.billed_until) + self.assertEqual(decimal.Decimal(0), bills[0].total) diff --git a/orchestra/contrib/services/tests/functional_tests/test_job.py b/orchestra/contrib/services/tests/functional_tests/test_job.py new file mode 100644 index 00000000..4e81d7ed --- /dev/null +++ b/orchestra/contrib/services/tests/functional_tests/test_job.py @@ -0,0 +1,49 @@ +from django.contrib.contenttypes.models import ContentType + +from orchestra.contrib.miscellaneous.models import MiscService, Miscellaneous +from orchestra.contrib.plans.models import Plan +from orchestra.utils.tests import random_ascii, BaseTestCase + +from ...models import Service + + +class JobBillingTest(BaseTestCase): + def create_job_service(self): + service = Service.objects.create( + description="Random job", + content_type=ContentType.objects.get_for_model(Miscellaneous), + match="miscellaneous.is_active and miscellaneous.service.name.lower() == 'job'", + billing_period=Service.NEVER, + billing_point=Service.ON_REGISTER, + is_fee=False, + metric='miscellaneous.amount', + pricing_period=Service.BILLING_PERIOD, + rate_algorithm='orchestra.contrib.plans.ratings.match_price', + on_cancel=Service.NOTHING, + payment_style=Service.POSTPAY, + tax=0, + nominal_price=20 + ) + plan = Plan.objects.create(is_default=True, name='Default') + service.rates.create(plan=plan, quantity=1, price=20) + service.rates.create(plan=plan, quantity=10, price=15) + return service + + def create_job(self, amount, account=None): + if not account: + account = self.create_account() + description = 'Random Job %s' % random_ascii(10) + service, __ = MiscService.objects.get_or_create(name='job', has_amount=True) + return account.miscellaneous.create(service=service, description=description, amount=amount) + + def test_job(self): + self.create_job_service() + account = self.create_account() + + self.create_job(5, account=account) + bill = account.orders.bill()[0] + self.assertEqual(5*20, bill.total) + + self.create_job(100, account=account) + bill = account.orders.bill(new_open=True)[0] + self.assertEqual(100*15, bill.total) diff --git a/orchestra/contrib/services/tests/functional_tests/test_mailbox.py b/orchestra/contrib/services/tests/functional_tests/test_mailbox.py new file mode 100644 index 00000000..5756cf62 --- /dev/null +++ b/orchestra/contrib/services/tests/functional_tests/test_mailbox.py @@ -0,0 +1,176 @@ +from dateutil.relativedelta import relativedelta +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone +from freezegun import freeze_time + +from orchestra.contrib.mailboxes.models import Mailbox +from orchestra.contrib.plans.models import Plan +from orchestra.contrib.resources.models import Resource, ResourceData +from orchestra.utils.tests import random_ascii, BaseTestCase + +from ...models import Service + + +class MailboxBillingTest(BaseTestCase): + def create_mailbox_service(self): + service = Service.objects.create( + description="Mailbox", + content_type=ContentType.objects.get_for_model(Mailbox), + match="True", + billing_period=Service.ANUAL, + billing_point=Service.FIXED_DATE, + is_fee=False, + metric='', + pricing_period=Service.NEVER, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.COMPENSATE, + payment_style=Service.PREPAY, + tax=0, + nominal_price=10 + ) + plan = Plan.objects.create(is_default=True, name='Default') + service.rates.create(plan=plan, quantity=1, price=0) + service.rates.create(plan=plan, quantity=5, price=10) + return service + + def create_mailbox_disk_service(self): + service = Service.objects.create( + description="Mailbox disk", + content_type=ContentType.objects.get_for_model(Mailbox), + match="True", + billing_period=Service.ANUAL, + billing_point=Service.FIXED_DATE, + is_fee=False, + metric='max((mailbox.resources.disk.allocated or 0) -1, 0)', + pricing_period=Service.NEVER, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.DISCOUNT, + payment_style=Service.PREPAY, + tax=0, + nominal_price=10 + ) + plan, __ = Plan.objects.get_or_create(is_default=True, name='Default') + service.rates.create(plan=plan, quantity=1, price=10) + return service + + def create_disk_resource(self): + self.resource = Resource.objects.create( + name='disk', + content_type=ContentType.objects.get_for_model(Mailbox), + aggregation='last', + verbose_name='Mailbox disk', + unit='GB', + scale=10**9, + on_demand=False, + monitors='MaildirDisk', + ) + return self.resource + + def allocate_disk(self, mailbox, value): + data, __ = ResourceData.objects.get_or_create(mailbox, self.resource) + data.allocated = value + data.save() + + def create_mailbox(self, account=None): + if not account: + account = self.create_account() + mailbox_name = '%s@orchestra.lan' % random_ascii(10) + return Mailbox.objects.create(name=mailbox_name, account=account) + + def test_mailbox_size(self): + service = self.create_mailbox_service() + disk_service = self.create_mailbox_disk_service() + self.create_disk_resource() + account = self.create_account() + mailbox = self.create_mailbox(account=account) + self.allocate_disk(mailbox, 10) + bill = service.orders.bill()[0] + self.assertEqual(0, bill.total) + bp = timezone.now().date() + relativedelta(years=1) + bill = disk_service.orders.bill(billing_point=bp, fixed_point=True)[0] + self.assertEqual(90, bill.total) + mailbox = self.create_mailbox(account=account) + mailbox = self.create_mailbox(account=account) + mailbox = self.create_mailbox(account=account) + mailbox = self.create_mailbox(account=account) + mailbox = self.create_mailbox(account=account) + mailbox = self.create_mailbox(account=account) + bill = service.orders.bill(billing_point=bp, fixed_point=True)[0] + self.assertEqual(120, bill.total) + + def test_mailbox_size_with_changes(self): + service = self.create_mailbox_disk_service() + self.create_disk_resource() + account = self.create_account() + mailbox = self.create_mailbox(account=account) + now = timezone.now() + bp = now.date() + relativedelta(years=1) + options = dict(billing_point=bp, fixed_point=True, proforma=True, new_open=True) + + self.allocate_disk(mailbox, 10) + bill = service.orders.bill(**options).pop() + self.assertEqual(9*10, bill.total) + + with freeze_time(now+relativedelta(months=6)): + self.allocate_disk(mailbox, 20) + bill = service.orders.bill(**options).pop() + total = 9*10*0.5 + 19*10*0.5 + self.assertEqual(total, bill.total) + + with freeze_time(now+relativedelta(months=9)): + self.allocate_disk(mailbox, 30) + bill = service.orders.bill(**options).pop() + total = 9*10*0.5 + 19*10*0.25 + 29*10*0.25 + self.assertEqual(total, bill.total) + + with freeze_time(now+relativedelta(years=1)): + self.allocate_disk(mailbox, 10) + bill = service.orders.bill(**options).pop() + total = 9*10*0.5 + 19*10*0.25 + 29*10*0.25 + self.assertEqual(total, bill.total) + + def test_mailbox_with_recharge(self): + service = self.create_mailbox_disk_service() + self.create_disk_resource() + account = self.create_account() + mailbox = self.create_mailbox(account=account) + now = timezone.now() + bp = now.date() + relativedelta(years=1) + options = dict(billing_point=bp, fixed_point=True) + + self.allocate_disk(mailbox, 100) + bill = service.orders.bill(**options).pop() + self.assertEqual(99*10, bill.total) + + with freeze_time(now+relativedelta(months=6)): + bills = service.orders.bill(new_open=True, **options) + self.assertEqual([], bills) + + with freeze_time(now+relativedelta(months=6)): + self.allocate_disk(mailbox, 50) + bills = service.orders.bill(**options) + self.assertEqual([], bills) + + with freeze_time(now+relativedelta(months=6)): + self.allocate_disk(mailbox, 200) + bill = service.orders.bill(new_open=True, **options).pop() + self.assertEqual((199-99)*10*0.5, bill.total) + + + def test_mailbox_second_billing(self): + service = self.create_mailbox_disk_service() + self.create_disk_resource() + account = self.create_account() + mailbox = self.create_mailbox(account=account) + now = timezone.now() + bp = now.date() + relativedelta(years=1) + options = dict(billing_point=bp, fixed_point=True) + bills = service.orders.bill(**options) + + with freeze_time(now+relativedelta(years=1, months=1)): + mailbox = self.create_mailbox(account=account) + alt_now = timezone.now() + bp = alt_now.date() + relativedelta(years=1) + options = dict(billing_point=bp, fixed_point=True) + bills = service.orders.bill(**options) + print(bills) diff --git a/orchestra/contrib/services/tests/functional_tests/test_plan.py b/orchestra/contrib/services/tests/functional_tests/test_plan.py new file mode 100644 index 00000000..d6c98af0 --- /dev/null +++ b/orchestra/contrib/services/tests/functional_tests/test_plan.py @@ -0,0 +1,52 @@ +from django.contrib.contenttypes.models import ContentType + +from orchestra.contrib.plans.models import Plan, ContractedPlan +from orchestra.utils.tests import BaseTestCase + +from ...models import Service + + +class PlanBillingTest(BaseTestCase): + def create_plan_service(self): + service = Service.objects.create( + description="Association membership fee", + content_type=ContentType.objects.get_for_model(ContractedPlan), + match="contractedplan.plan.name == 'association_fee'", + billing_period=Service.ANUAL, + billing_point=Service.FIXED_DATE, + is_fee=True, + metric='', + pricing_period=Service.BILLING_PERIOD, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.DISCOUNT, + payment_style=Service.PREPAY, + tax=0, + nominal_price=20 + ) + return service + + def create_plan(self, account=None): + if not account: + account = self.create_account() + plan, __ = Plan.objects.get_or_create(name='association_fee') + return plan.contracts.create(account=account) + + def test_update_orders(self): + account = self.create_account() + account1 = self.create_account() + self.create_plan(account=account) + self.create_plan(account=account1) + service = self.create_plan_service() + self.assertEqual(0, service.orders.count()) + service.update_orders() + self.assertEqual(2, service.orders.count()) + + def test_plan(self): + account = self.create_account() + self.create_plan_service() + self.create_plan(account=account) + bill = account.orders.bill().pop() + self.assertEqual(bill.FEE, bill.type) + + +# TODO test price with multiple plans diff --git a/orchestra/contrib/services/tests/functional_tests/test_traffic.py b/orchestra/contrib/services/tests/functional_tests/test_traffic.py new file mode 100644 index 00000000..e7334de9 --- /dev/null +++ b/orchestra/contrib/services/tests/functional_tests/test_traffic.py @@ -0,0 +1,169 @@ +from dateutil.relativedelta import relativedelta +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone +from freezegun import freeze_time + +from orchestra.contrib.accounts.models import Account +from orchestra.contrib.miscellaneous.models import MiscService, Miscellaneous +from orchestra.contrib.plans.models import Plan +from orchestra.contrib.resources.models import Resource, ResourceData, MonitorData +from orchestra.contrib.resources.backends import ServiceMonitor +from orchestra.utils.tests import BaseTestCase + +from ...models import Service + + +class FTPTrafficMonitor(ServiceMonitor): + model = 'systemusers.SystemUser' + + +class BaseTrafficBillingTest(BaseTestCase): + TRAFFIC_METRIC = 'account.resources.traffic.used' + DEPENDENCIES = ('orchestra.contrib.resources',) + + def create_traffic_service(self): + service = Service.objects.create( + description="Traffic", + content_type=ContentType.objects.get_for_model(Account), + match="account.is_active", + billing_period=Service.MONTHLY, + billing_point=Service.FIXED_DATE, + is_fee=False, + metric=self.TRAFFIC_METRIC, + pricing_period=Service.BILLING_PERIOD, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.NOTHING, + payment_style=Service.POSTPAY, + tax=0, + nominal_price=10 + ) + plan = Plan.objects.create(is_default=True, name='Default') + service.rates.create(plan=plan, quantity=1, price=0) + service.rates.create(plan=plan, quantity=11, price=10) + return service + + def create_traffic_resource(self): + self.resource = Resource.objects.create( + name='traffic', + content_type=ContentType.objects.get_for_model(Account), + aggregation='monthly-sum', + verbose_name='Account Traffic', + unit='GB', + scale='10**9', + on_demand=True, + # TODO + monitors=[FTPTrafficMonitor.get_name()], + ) + return self.resource + + def report_traffic(self, account, value): + MonitorData.objects.create(monitor=FTPTrafficMonitor.get_name(), content_object=account.systemusers.get(), value=value) + data, __ = ResourceData.objects.get_or_create(account, self.resource) + data.update() + + +class TrafficBillingTest(BaseTrafficBillingTest): + def test_traffic(self): + self.create_traffic_service() + self.create_traffic_resource() + account = self.create_account() + now = timezone.now() + + self.report_traffic(account, 10**9) + bill = account.orders.bill(commit=False)[0] + self.assertEqual((account, []), bill) + self.report_traffic(account, 10**9*9) + + with freeze_time(now+relativedelta(months=1)): + bill = account.orders.bill(proforma=True)[0] + self.report_traffic(account, 10**10*9) + self.assertEqual(0, bill.total) + + with freeze_time(now+relativedelta(months=3)): + bill = account.orders.bill(proforma=True)[0] + self.assertEqual((90-10)*10, bill.total) + + def test_multiple_traffics(self): + self.create_traffic_service() + self.create_traffic_resource() + account1 = self.create_account() + account2 = self.create_account() + self.report_traffic(account1, 10**10) + self.report_traffic(account2, 10**10*5) + with freeze_time(timezone.now()+relativedelta(months=1)): + bill1 = account1.orders.bill().pop() + bill2 = account2.orders.bill().pop() + self.assertNotEqual(bill1.total, bill2.total) + + +class TrafficPrepayBillingTest(BaseTrafficBillingTest): + TRAFFIC_METRIC = ("max(" + "(account.resources.traffic.used or 0) - " + "getattr(account.miscellaneous.filter(is_active=True, service__name='traffic prepay').last(), 'amount', 0)" + ", 0)" + ) + + def create_prepay_service(self): + service = Service.objects.create( + description="Traffic prepay", + content_type=ContentType.objects.get_for_model(Miscellaneous), + match="miscellaneous.is_active and miscellaneous.service.name.lower() == 'traffic prepay'", + billing_period=Service.MONTHLY, + # make sure full months are always paid + billing_point=Service.ON_REGISTER, + is_fee=False, + metric="miscellaneous.amount", + pricing_period=Service.NEVER, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.NOTHING, + payment_style=Service.PREPAY, + tax=0, + nominal_price=50 + ) + return service + + def create_prepay(self, amount, account=None): + if not account: + account = self.create_account() + name = 'traffic prepay' + service, __ = MiscService.objects.get_or_create(name='traffic prepay', + description='Traffic prepay', has_amount=True) + return account.miscellaneous.create(service=service, description=name, amount=amount) + + def test_traffic_prepay(self): + self.create_traffic_service() + self.create_prepay_service() + self.create_traffic_resource() + account = self.create_account() + now = timezone.now() + + self.create_prepay(10, account=account) + bill = account.orders.bill(proforma=True)[0] + self.assertEqual(10*50, bill.total) + + self.report_traffic(account, 10**10) + with freeze_time(now+relativedelta(months=1)): + bill = account.orders.bill(proforma=True, new_open=True)[0] + self.assertEqual(2*10*50 + 0*10, bill.total) + + # TODO RuntimeWarning: DateTimeField MetricStorage.updated_on received a naive + self.report_traffic(account, 10**10) + with freeze_time(now+relativedelta(months=1)): + bill = account.orders.bill(proforma=True, new_open=True)[0] + self.assertEqual(2*10*50 + 0*10, bill.total) + + self.report_traffic(account, 10**10) + with freeze_time(now+relativedelta(months=1)): + bill = account.orders.bill(proforma=True, new_open=True)[0] + self.assertEqual(2*10*50 + (30-10-10)*10, bill.total) + + with freeze_time(now+relativedelta(months=2)): + self.report_traffic(account, 10**11) + with freeze_time(now+relativedelta(months=1)): + bill = account.orders.bill(proforma=True, new_open=True)[0] + self.assertEqual(2*10*50 + (30-10-10)*10, bill.total) + + with freeze_time(now+relativedelta(months=3)): + bill = account.orders.bill(proforma=True, new_open=True)[0] + self.assertEqual(4*10*50 + (30-10-10)*10 + (100-10-10)*10, bill.total) + diff --git a/orchestra/contrib/services/tests/test_handler.py b/orchestra/contrib/services/tests/test_handler.py new file mode 100644 index 00000000..17d5d154 --- /dev/null +++ b/orchestra/contrib/services/tests/test_handler.py @@ -0,0 +1,537 @@ +import datetime +import decimal +from functools import cmp_to_key + +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone + +from orchestra.contrib.systemusers.models import SystemUser +from orchestra.contrib.plans.models import Plan +from orchestra.utils.tests import BaseTestCase + +from .. import helpers +from ..models import Service + + +class Order(object): + """ Fake order for testing """ + last_id = 0 + + def __init__(self, **kwargs): + self.registered_on = kwargs.get('registered_on') or timezone.now().date() + self.billed_until = kwargs.get('billed_until', None) + self.cancelled_on = kwargs.get('cancelled_on', None) + type(self).last_id += 1 + self.id = self.last_id + self.pk = self.id + + +class HandlerTests(BaseTestCase): + DEPENDENCIES = ( + 'orchestra.contrib.orders', + 'orchestra.contrib.systemusers', + ) + + def create_ftp_service(self, **kwargs): + default = dict( + description="FTP Account", + content_type=ContentType.objects.get_for_model(SystemUser), + match='not systemuser.is_main', + billing_period=Service.ANUAL, + billing_point=Service.FIXED_DATE, + is_fee=False, + metric='', + pricing_period=Service.NEVER, + rate_algorithm='orchestra.contrib.plans.ratings.step_price', + on_cancel=Service.DISCOUNT, + payment_style=Service.PREPAY, + tax=0, + nominal_price=10 + ) + default.update(kwargs) + service = Service.objects.create(**default) + return service + + def validate_results(self, rates, results): + self.assertEqual(len(rates), len(results)) + for rate, result in zip(rates, results): + self.assertEqual(rate['price'], result.price) + self.assertEqual(rate['quantity'], result.quantity) + + def test_get_chunks(self): + service = self.create_ftp_service() + handler = service.handler + porders = [] + now = timezone.now().date() + + order = Order() + porders.append(order) + end = handler.get_billing_point(order) + chunks = helpers.get_chunks(porders, now, end) + self.assertEqual(1, len(chunks)) + self.assertIn([now, end, []], chunks) + + order1 = Order( + billed_until=now+datetime.timedelta(days=2) + ) + porders.append(order1) + chunks = helpers.get_chunks(porders, now, end) + self.assertEqual(2, len(chunks)) + self.assertIn([order1.registered_on, order1.billed_until, [order1]], chunks) + self.assertIn([order1.billed_until, end, []], chunks) + + order2 = Order( + billed_until = now+datetime.timedelta(days=700) + ) + porders.append(order2) + chunks = helpers.get_chunks(porders, now, end) + self.assertEqual(2, len(chunks)) + self.assertIn([order.registered_on, order1.billed_until, [order1, order2]], chunks) + self.assertIn([order1.billed_until, end, [order2]], chunks) + + order3 = Order( + billed_until = now+datetime.timedelta(days=700) + ) + porders.append(order3) + chunks = helpers.get_chunks(porders, now, end) + self.assertEqual(2, len(chunks)) + self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) + self.assertIn([order1.billed_until, end, [order2, order3]], chunks) + + order4 = Order( + registered_on=now+datetime.timedelta(days=5), + billed_until = now+datetime.timedelta(days=10) + ) + porders.append(order4) + chunks = helpers.get_chunks(porders, now, end) + self.assertEqual(4, len(chunks)) + self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) + self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks) + self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks) + self.assertIn([order4.billed_until, end, [order2, order3]], chunks) + + order5 = Order( + registered_on=now+datetime.timedelta(days=700), + billed_until=now+datetime.timedelta(days=780) + ) + porders.append(order5) + chunks = helpers.get_chunks(porders, now, end) + self.assertEqual(4, len(chunks)) + self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) + self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks) + self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks) + self.assertIn([order4.billed_until, end, [order2, order3]], chunks) + + order6 = Order( + registered_on=now+datetime.timedelta(days=780), + billed_until=now+datetime.timedelta(days=700) + ) + porders.append(order6) + chunks = helpers.get_chunks(porders, now, end) + self.assertEqual(4, len(chunks)) + self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) + self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks) + self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks) + self.assertIn([order4.billed_until, end, [order2, order3]], chunks) + + def test_sort_billed_until_or_registered_on(self): + now = timezone.now().date() + order = Order( + billed_until=now+datetime.timedelta(days=200)) + order1 = Order( + registered_on=now+datetime.timedelta(days=5), + billed_until=now+datetime.timedelta(days=200)) + order2 = Order( + registered_on=now+datetime.timedelta(days=6), + billed_until=now+datetime.timedelta(days=200)) + order3 = Order( + registered_on=now+datetime.timedelta(days=6), + billed_until=now+datetime.timedelta(days=201)) + order4 = Order( + registered_on=now+datetime.timedelta(days=6)) + order5 = Order( + registered_on=now+datetime.timedelta(days=7)) + order6 = Order( + registered_on=now+datetime.timedelta(days=8)) + orders = [order3, order, order1, order2, order4, order5, order6] + self.assertEqual(orders, sorted(orders, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on))) + + def test_compensation(self): + now = timezone.now().date() + order = Order( + description='0', + billed_until=now+datetime.timedelta(days=220), + cancelled_on=now+datetime.timedelta(days=100)) + order1 = Order( + description='1', + registered_on=now+datetime.timedelta(days=5), + cancelled_on=now+datetime.timedelta(days=190), + billed_until=now+datetime.timedelta(days=200)) + order2 = Order( + description='2', + registered_on=now+datetime.timedelta(days=6), + cancelled_on=now+datetime.timedelta(days=200), + billed_until=now+datetime.timedelta(days=200)) + order3 = Order( + description='3', + registered_on=now+datetime.timedelta(days=6), + billed_until=now+datetime.timedelta(days=200)) + + tests = [] + order4 = Order( + description='4', + registered_on=now+datetime.timedelta(days=6), + billed_until=now+datetime.timedelta(days=102)) + order4.new_billed_until = now+datetime.timedelta(days=200) + tests.append([ + [now+datetime.timedelta(days=102), now+datetime.timedelta(days=220), order], + ]) + order5 = Order( + description='5', + registered_on=now+datetime.timedelta(days=7), + billed_until=now+datetime.timedelta(days=102)) + order5.new_billed_until = now+datetime.timedelta(days=195) + tests.append([ + [now+datetime.timedelta(days=190), now+datetime.timedelta(days=200), order1] + ]) + order6 = Order( + description='6', + registered_on=now+datetime.timedelta(days=8)) + order6.new_billed_until = now+datetime.timedelta(days=200) + tests.append([ + [now+datetime.timedelta(days=100), now+datetime.timedelta(days=102), order], + ]) + porders = [order3, order, order1, order2, order4, order5, order6] + porders = sorted(porders, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on)) + compensations = [] + receivers = [] + for order in porders: + if order.billed_until and order.cancelled_on and order.cancelled_on < order.billed_until: + compensations.append(helpers.Interval(order.cancelled_on, order.billed_until, order=order)) + elif hasattr(order, 'new_billed_until') and (not order.billed_until or order.billed_until < order.new_billed_until): + receivers.append(order) + for order, test in zip(receivers, tests): + ini = order.billed_until or order.registered_on + end = order.cancelled_on or now+datetime.timedelta(days=20000) + order_interval = helpers.Interval(ini, end) + (compensations, used_compensations) = helpers.compensate(order_interval, compensations) + for compensation, test_line in zip(used_compensations, test): + self.assertEqual(test_line[0], compensation.ini) + self.assertEqual(test_line[1], compensation.end) + self.assertEqual(test_line[2], compensation.order) + + def test_rates(self): + service = self.create_ftp_service() + account = self.create_account() + superplan = Plan.objects.create( + name='SUPER', allow_multiple=False, is_combinable=True) + service.rates.create(plan=superplan, quantity=0, price=0) + service.rates.create(plan=superplan, quantity=3, price=10) + service.rates.create(plan=superplan, quantity=4, price=9) + service.rates.create(plan=superplan, quantity=10, price=1) + account.plans.create(plan=superplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 2 + }, { + 'price': decimal.Decimal('10.00'), + 'quantity': 1 + }, { + 'price': decimal.Decimal('9.00'), + 'quantity': 6 + }, { + 'price': decimal.Decimal('1.00'), + 'quantity': 21 + } + ] + self.validate_results(rates, results) + + dupeplan = Plan.objects.create( + name='DUPE', allow_multiple=True, is_combinable=True) + service.rates.create(plan=dupeplan, quantity=1, price=0) + service.rates.create(plan=dupeplan, quantity=3, price=9) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + self.validate_results(rates, results) + + account.plans.create(plan=dupeplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 4}, + {'price': decimal.Decimal('9.00'), 'quantity': 5}, + {'price': decimal.Decimal('1.00'), 'quantity': 21}, + ] + self.validate_results(rates, results) + + hyperplan = Plan.objects.create( + name='HYPER', allow_multiple=False, is_combinable=False) + service.rates.create(plan=hyperplan, quantity=0, price=0) + service.rates.create(plan=hyperplan, quantity=20, price=5) + account.plans.create(plan=hyperplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 19}, + {'price': decimal.Decimal('5.00'), 'quantity': 11} + ] + self.validate_results(rates, results) + + hyperplan.is_combinable = True + hyperplan.save() + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 23}, + {'price': decimal.Decimal('1.00'), 'quantity': 7} + ] + self.validate_results(rates, results) + + service.rate_algorithm = 'orchestra.contrib.plans.ratings.match_price' + service.save() + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + self.assertEqual(1, len(results)) + self.assertEqual(decimal.Decimal('1.00'), results[0].price) + self.assertEqual(30, results[0].quantity) + + hyperplan.delete() + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 8) + self.assertEqual(1, len(results)) + self.assertEqual(decimal.Decimal('9.00'), results[0].price) + self.assertEqual(8, results[0].quantity) + + superplan.delete() + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + self.assertEqual(1, len(results)) + self.assertEqual(decimal.Decimal('9.00'), results[0].price) + self.assertEqual(30, results[0].quantity) + + def test_incomplete_rates(self): + service = self.create_ftp_service() + account = self.create_account() + superplan = Plan.objects.create( + name='SUPER', allow_multiple=False, is_combinable=True) + service.rates.create(plan=superplan, quantity=4, price=9) + service.rates.create(plan=superplan, quantity=10, price=1) + account.plans.create(plan=superplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + { + 'price': decimal.Decimal('10.00'), + 'quantity': 3 + }, { + 'price': decimal.Decimal('9.00'), + 'quantity': 6 + }, { + 'price': decimal.Decimal('1.00'), + 'quantity': 21 + } + ] + self.validate_results(rates, results) + + def test_zero_rates(self): + service = self.create_ftp_service() + account = self.create_account() + superplan = Plan.objects.create( + name='SUPER', allow_multiple=False, is_combinable=True) + service.rates.create(plan=superplan, quantity=0, price=0) + service.rates.create(plan=superplan, quantity=3, price=10) + service.rates.create(plan=superplan, quantity=4, price=9) + service.rates.create(plan=superplan, quantity=10, price=1) + account.plans.create(plan=superplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 2}, + {'price': decimal.Decimal('10.00'), 'quantity': 1}, + {'price': decimal.Decimal('9.00'), 'quantity': 6}, + {'price': decimal.Decimal('1.00'), 'quantity': 21} + ] + self.validate_results(rates, results) + + def test_rates_allow_multiple(self): + service = self.create_ftp_service() + account = self.create_account() + dupeplan = Plan.objects.create( + name='DUPE', allow_multiple=True, is_combinable=True) + account.plans.create(plan=dupeplan) + service.rates.create(plan=dupeplan, quantity=0, price=0) + service.rates.create(plan=dupeplan, quantity=3, price=9) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 2}, + {'price': decimal.Decimal('9.00'), 'quantity': 28}, + ] + self.validate_results(rates, results) + + account.plans.create(plan=dupeplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 4}, + {'price': decimal.Decimal('9.00'), 'quantity': 26}, + ] + self.validate_results(rates, results) + + account.plans.create(plan=dupeplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 6}, + {'price': decimal.Decimal('9.00'), 'quantity': 24}, + ] + self.validate_results(rates, results) + + def test_best_price(self): + service = self.create_ftp_service(rate_algorithm='orchestra.contrib.plans.ratings.best_price') + account = self.create_account() + dupeplan = Plan.objects.create(name='DUPE') + account.plans.create(plan=dupeplan) + service.rates.create(plan=dupeplan, quantity=0, price=0) + service.rates.create(plan=dupeplan, quantity=2, price=9) + service.rates.create(plan=dupeplan, quantity=3, price=8) + service.rates.create(plan=dupeplan, quantity=4, price=7) + service.rates.create(plan=dupeplan, quantity=5, price=10) + service.rates.create(plan=dupeplan, quantity=10, price=5) + raw_rates = service.get_rates(account, cache=False) + results = service.rate_method(raw_rates, 2) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('9.00'), + 'quantity': 1 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 3) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('8.00'), + 'quantity': 2 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 5) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('7.00'), + 'quantity': 4 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 9) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('7.00'), + 'quantity': 4 + }, + { + 'price': decimal.Decimal('10.00'), + 'quantity': 4 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 10) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 1 + }, + { + 'price': decimal.Decimal('5.00'), + 'quantity': 9 + }, + ] + self.validate_results(rates, results) + + def test_best_price_multiple(self): + service = self.create_ftp_service(rate_algorithm='orchestra.contrib.plans.ratings.best_price') + account = self.create_account() + dupeplan = Plan.objects.create(name='DUPE') + account.plans.create(plan=dupeplan) + account.plans.create(plan=dupeplan) + service.rates.create(plan=dupeplan, quantity=0, price=0) + service.rates.create(plan=dupeplan, quantity=2, price=9) + service.rates.create(plan=dupeplan, quantity=3, price=8) + service.rates.create(plan=dupeplan, quantity=4, price=7) + service.rates.create(plan=dupeplan, quantity=5, price=10) + service.rates.create(plan=dupeplan, quantity=10, price=5) + raw_rates = service.get_rates(account, cache=False) + + results = service.rate_method(raw_rates, 3) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 2 + }, + { + 'price': decimal.Decimal('8.00'), + 'quantity': 1 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 10) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 2 + }, + { + 'price': decimal.Decimal('5.00'), + 'quantity': 8 + }, + ] + self.validate_results(rates, results) + + account.plans.create(plan=dupeplan) + raw_rates = service.get_rates(account, cache=False) + + results = service.rate_method(raw_rates, 3) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 3 + }, + ] + self.validate_results(rates, results) + + results = service.rate_method(raw_rates, 10) + rates = [ + { + 'price': decimal.Decimal('0.00'), + 'quantity': 3 + }, + { + 'price': decimal.Decimal('5.00'), + 'quantity': 7 + }, + ] + self.validate_results(rates, results) diff --git a/orchestra/contrib/settings/README.md b/orchestra/contrib/settings/README.md new file mode 100644 index 00000000..4de7305b --- /dev/null +++ b/orchestra/contrib/settings/README.md @@ -0,0 +1,18 @@ +```python +>>> from orchestra.contrib.settings import Setting, parser +>>> Setting.settings['TASKS_BACKEND'].value +'thread' +>>> Setting.settings['TASKS_BACKEND'].default +'thread' +>>> Setting.settings['TASKS_BACKEND'].validate_value('rata') +Traceback (most recent call last): + File "", line 1, in + File "/home/orchestra/django-orchestra/orchestra/contrib/settings/__init__.py", line 99, in validate_value + raise ValidationError("'%s' not in '%s'" % (value, ', '.join(choices))) +django.core.exceptions.ValidationError: ["'rata' not in 'thread, process, celery'"] +>>> parser.apply({'TASKS_BACKEND': 'process'}) +... +>>> parser.apply({'TASKS_BACKEND': parser.Remove()}) +... +``` + diff --git a/orchestra/contrib/settings/__init__.py b/orchestra/contrib/settings/__init__.py new file mode 100644 index 00000000..1dc831a5 --- /dev/null +++ b/orchestra/contrib/settings/__init__.py @@ -0,0 +1,113 @@ +import re +from collections import OrderedDict + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.utils.functional import Promise + +from orchestra.core import validators +from orchestra.utils.python import import_class, format_exception + + +default_app_config = 'orchestra.contrib.settings.apps.SettingsConfig' + + +class Setting(object): + """ + Keeps track of the defined settings and provides extra batteries like value validation. + """ + conf_settings = settings + settings = OrderedDict() + + def __str__(self): + return self.name + + def __repr__(self): + value = str(self.value) + value = ("'%s'" if isinstance(value, str) else '%s') % value + return '<%s: %s>' % (self.name, value) + + def __new__(cls, name, default, help_text="", choices=None, editable=True, serializable=True, + multiple=False, validators=[], types=[], call_init=False): + if call_init: + return super(Setting, cls).__new__(cls) + cls.settings[name] = cls(name, default, help_text=help_text, choices=choices, editable=editable, + serializable=serializable, multiple=multiple, validators=validators, types=types, call_init=True) + return cls.get_value(name, default) + + def __init__(self, *args, **kwargs): + self.name, self.default = args + for name, value in kwargs.items(): + setattr(self, name, value) + self.value = self.get_value(self.name, self.default) + self.settings[name] = self + + @classmethod + def validate_choices(cls, value): + if not isinstance(value, (list, tuple)): + raise ValidationError("%s is not a valid choices." % value) + for choice in value: + if not isinstance(choice, (list, tuple)) or len(choice) != 2: + raise ValidationError("%s is not a valid choice." % choice) + value, verbose = choice + if not isinstance(verbose, (str, Promise)): + raise ValidationError("%s is not a valid verbose name." % value) + + @classmethod + def validate_import_class(cls, value): + try: + import_class(value) + except Exception as exc: + raise ValidationError(format_exception(exc)) + + @classmethod + def validate_model_label(cls, value): + from django.apps import apps + try: + apps.get_model(*value.split('.')) + except Exception as exc: + raise ValidationError(format_exception(exc)) + + @classmethod + def string_format_validator(cls, names, modulo=True): + def validate_string_format(value, names=names, modulo=modulo): + errors = [] + regex = r'%\(([^\)]+)\)' if modulo else r'{([^}]+)}' + for n in re.findall(regex, value): + if n not in names: + errors.append( + ValidationError('%s is not a valid format name.' % n) + ) + if errors: + raise ValidationError(errors) + return validate_string_format + + def validate_value(self, value): + if value: + validators.all_valid(value, self.validators) + valid_types = list(self.types) + if self.choices: + choices = self.choices + if callable(choices): + choices = choices() + choices = [n for n,v in choices] + values = value + if not isinstance(values, (list, tuple)): + values = [value] + for cvalue in values: + if cvalue not in choices: + raise ValidationError("'%s' not in '%s'" % (value, ', '.join(choices))) + if isinstance(self.default, (list, tuple)): + valid_types.extend([list, tuple]) + valid_types.append(type(self.default)) + if not isinstance(value, tuple(valid_types)): + raise ValidationError("%s is not a valid type (%s)." % + (type(value).__name__, ', '.join(t.__name__ for t in valid_types)) + ) + + def validate(self): + self.validate_value(self.value) + + @classmethod + def get_value(cls, name, default): + return getattr(cls.conf_settings, name, default) diff --git a/orchestra/contrib/settings/admin.py b/orchestra/contrib/settings/admin.py new file mode 100644 index 00000000..6cefcb08 --- /dev/null +++ b/orchestra/contrib/settings/admin.py @@ -0,0 +1,110 @@ +from django.contrib import admin, messages +from django.shortcuts import render +from django.views import generic +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.contrib.settings import Setting +from orchestra.utils import sys + +from . import parser +from .forms import SettingFormSet + + +class SettingView(generic.edit.FormView): + template_name = 'admin/settings/change_form.html' + reload_template_name = 'admin/settings/reload.html' + form_class = SettingFormSet + success_url = '.' + + def get_context_data(self, **kwargs): + context = super(SettingView, self).get_context_data(**kwargs) + context.update({ + 'title': _("Change settings"), + 'settings_file': parser.get_settings_file(), + }) + return context + + def get_initial(self): + initial_data = [] + prev_app = None + account = 0 + for name, setting in Setting.settings.items(): + app = name.split('_')[0] + initial = { + 'name': setting.name, + 'help_text': setting.help_text, + 'default': setting.default, + 'type': type(setting.default), + 'value': setting.value, + 'setting': setting, + 'app': app, + } + if app == 'ORCHESTRA': + initial_data.insert(account, initial) + account += 1 + else: + initial_data.append(initial) + return initial_data + + def form_valid(self, form): + settings = Setting.settings + changes = {} + for data in form.cleaned_data: + setting = settings[data['name']] + if not isinstance(data['value'], parser.NotSupported) and setting.editable: + if setting.value != data['value']: + # Ignore differences between lists and tuples + if (type(setting.value) != type(data['value']) and + isinstance(data['value'], list) and + tuple(data['value']) == setting.value): + continue + if setting.default == data['value']: + changes[setting.name] = parser.Remove() + else: + changes[setting.name] = data['value'] + if changes: + # Display confirmation + if not self.request.POST.get('confirmation'): + settings_file = parser.get_settings_file() + new_content = parser.apply(changes) + cmd = "cat < + +{% endblock %} + + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +{% if diff %} › Settings › Confirm changes{% else %} › Settings{% endif %} +
    +{% endblock %} + +{% block content %} + + +
    +
    {% csrf_token %} + {% if diff %} + {% blocktrans %} +

    The following changes will be performed to {{ settings_file }} file.

    + {% endblocktrans %} + 
    {{ diff }}
    + {{ form.management_form }} + + {% for form in form %} + {{ form }} + {% endfor %} +
    + +
    + {% else %} + {% blocktrans %} +

    {{ settings_file }} file will be automatically updated and Orchestra restarted according to your changes. + {% endblocktrans %} + {% if form.errors %} +

    + {% trans "Please correct the errors below." %} +

    + {{ form.non_form_errors.as_ul }} + {% endif %} + {{ form.management_form }} + {% regroup form.forms by app as formlist %} + {% for app in formlist %} +
    +

    {{ app.grouper|lower|capfirst }}

    + + {% for form in app.list %} + {{ form.non_field_errors }} + {% if forloop.first %} + + {% for field in form.visible_fields %} + + {% endfor %} + + {% endif %} + + {% for field in form.visible_fields %} + + {% endfor %} + + {% endfor %} +
    {{ field.label|capfirst }}
    + {# Include the hidden fields in the form #} + {% if forloop.first %} + {% for hidden in form.hidden_fields %} + {{ hidden }} + {% endfor %} + {% endif %} + {{ field.errors.as_ul }} +
    {{ field }}{% if forloop.last %}{% if form.changed %}
    *
    {% endif %}{% endif %}
    +

    {{ field.help_text }}

    +
    +
    + {% endfor %} +
    + {% endif %} +
    +{% endblock %} diff --git a/orchestra/contrib/settings/templates/admin/settings/reload.html b/orchestra/contrib/settings/templates/admin/settings/reload.html new file mode 100644 index 00000000..7f183077 --- /dev/null +++ b/orchestra/contrib/settings/templates/admin/settings/reload.html @@ -0,0 +1,54 @@ +{% load static %} + + + + + + + + + +
    +
    notice: {{ message }}
    Refreshing in 2.
    +
    + + diff --git a/orchestra/contrib/settings/templates/admin/settings/view.html b/orchestra/contrib/settings/templates/admin/settings/view.html new file mode 100644 index 00000000..9f8ead55 --- /dev/null +++ b/orchestra/contrib/settings/templates/admin/settings/view.html @@ -0,0 +1,28 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n %} +{% load admin_urls static utils %} + +{% block extrastyle %} +{{ block.super }} + + +{% endblock %} + + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› Settings +› View file content +
    +{% endblock %} + + +{% block content %} +
    + {% blocktrans %} +

    Current {{ settings_file }} content.

    + {% endblocktrans %} + 
    {{ content }}
    +
    +{% endblock %} diff --git a/orchestra/contrib/systemusers/__init__.py b/orchestra/contrib/systemusers/__init__.py new file mode 100644 index 00000000..1fbedd5f --- /dev/null +++ b/orchestra/contrib/systemusers/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.systemusers.apps.SystemUsersConfig' diff --git a/orchestra/contrib/systemusers/actions.py b/orchestra/contrib/systemusers/actions.py new file mode 100644 index 00000000..1916aa47 --- /dev/null +++ b/orchestra/contrib/systemusers/actions.py @@ -0,0 +1,130 @@ +import os + +from django.contrib import messages, admin +from django.core.exceptions import PermissionDenied +from django.template.response import TemplateResponse +from django.utils.translation import ngettext, gettext_lazy as _ + +from orchestra.contrib.orchestration import Operation, helpers + +from .forms import PermissionForm, LinkForm + + +def get_verbose_choice(choices, value): + for choice, verbose in choices: + if choice == value: + return verbose + + +def set_permission(modeladmin, request, queryset): + account_id = None + for user in queryset: + account_id = account_id or user.account_id + if user.account_id != account_id: + messages.error(request, "Users from the same account should be selected.") + return + user = queryset[0] + form = PermissionForm(user) + action_value = 'set_permission' + if request.POST.get('post') == 'generic_confirmation': + form = PermissionForm(user, request.POST) + if form.is_valid(): + cleaned_data = form.cleaned_data + operations = [] + for user in queryset: + base_home = cleaned_data['base_home'] + extension = cleaned_data['home_extension'] + action = cleaned_data['set_action'] + perms = cleaned_data['permissions'] + user.set_perm_action = action + user.set_perm_base_home = base_home + user.set_perm_home_extension = extension + user.set_perm_perms = perms + operations.extend(Operation.create_for_action(user, 'set_permission')) + verbose_action = get_verbose_choice(form.fields['set_action'].choices, + user.set_perm_action) + verbose_permissions = get_verbose_choice(form.fields['permissions'].choices, + user.set_perm_perms) + context = { + 'action': verbose_action, + 'perms': verbose_permissions.lower(), + 'to': os.path.join(user.set_perm_base_home, user.set_perm_home_extension), + } + msg = _("%(action)s %(perms)s permission to %(to)s") % context + modeladmin.log_change(request, user, msg) + if not operations: + messages.error(request, _("No backend operation has been executed.")) + else: + logs = Operation.execute(operations) + helpers.message_user(request, logs) + return + opts = modeladmin.model._meta + app_label = opts.app_label + context = { + 'title': _("Set permission"), + 'action_name': _("Set permission"), + 'action_value': action_value, + 'queryset': queryset, + 'opts': opts, + 'obj': user, + 'app_label': app_label, + 'action_checkbox_name': admin.helpers.ACTION_CHECKBOX_NAME, + 'form': form, + } + return TemplateResponse(request, 'admin/systemusers/systemuser/set_permission.html', context) +set_permission.url_name = 'set-permission' +set_permission.tool_description = _("Set permission") + + +def create_link(modeladmin, request, queryset): + account_id = None + for user in queryset: + account_id = account_id or user.account_id + if user.account_id != account_id: + messages.error(request, "Users from the same account should be selected.") + return + user = queryset[0] + form = LinkForm(user, queryset=queryset) + action_value = 'create_link' + if request.POST.get('post') == 'generic_confirmation': + form = LinkForm(user, request.POST, queryset=queryset) + if form.is_valid(): + cleaned_data = form.cleaned_data + operations = [] + for user in queryset: + base_home = cleaned_data['base_home'] + extension = cleaned_data['home_extension'] + target = os.path.join(base_home, extension) + default_name = os.path.join(user.home, os.path.basename(target)) + link_name = cleaned_data['link_name'] or default_name + user.create_link_target = target + user.create_link_name = link_name + operations.extend(Operation.create_for_action(user, 'create_link')) + context = { + 'target': target, + 'link_name': link_name, + } + msg = _("Created link from %(target)s to %(link_name)s") % context + modeladmin.log_change(request, request.user, msg) + logs = Operation.execute(operations) + if logs: + helpers.message_user(request, logs) + else: + messages.error(request, "No backend operation has been executed.") + return + opts = modeladmin.model._meta + app_label = opts.app_label + context = { + 'title': _("Create link"), + 'action_name': _("Create link"), + 'action_value': action_value, + 'queryset': queryset, + 'opts': opts, + 'obj': user, + 'app_label': app_label, + 'action_checkbox_name': admin.helpers.ACTION_CHECKBOX_NAME, + 'form': form, + } + return TemplateResponse(request, 'admin/systemusers/systemuser/create_link.html', context) +create_link.url_name = 'create-link' +create_link.tool_description = _("Create link") diff --git a/orchestra/contrib/systemusers/admin.py b/orchestra/contrib/systemusers/admin.py new file mode 100644 index 00000000..64362957 --- /dev/null +++ b/orchestra/contrib/systemusers/admin.py @@ -0,0 +1,81 @@ +from django.contrib import admin, messages +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin +from orchestra.admin.actions import disable, enable +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import SelectAccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter + +from .actions import set_permission, create_link +from .filters import IsMainListFilter +from .forms import SystemUserCreationForm, SystemUserChangeForm +from .models import SystemUser + + +class SystemUserAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'username', 'account_link', 'shell', 'display_home', 'display_active', 'display_main' + ) + list_filter = (IsActiveListFilter, 'shell', IsMainListFilter) + fieldsets = ( + (None, { + 'fields': ('username', 'password', 'account_link', 'is_active') + }), + (_("System"), { + 'fields': ('shell', ('home', 'directory'), 'groups'), + }), + ) + add_fieldsets = ( + (None, { + 'fields': ('account_link', 'username', 'password1', 'password2') + }), + (_("System"), { + 'fields': ('shell', ('home', 'directory'), 'groups'), + }), + ) + search_fields = ('username', 'account__username') + readonly_fields = ('account_link',) + change_readonly_fields = ('username',) + filter_horizontal = ('groups',) + filter_by_account_fields = ('groups',) + add_form = SystemUserCreationForm + form = SystemUserChangeForm + ordering = ('-id',) + change_view_actions = (set_permission, create_link) + actions = (disable, enable, list_accounts) + change_view_actions + + def display_main(self, user): + return user.is_main + display_main.short_description = _("Main") + display_main.boolean = True + + def display_home(self, user): + return user.get_home() + display_home.short_description = _("Home") + display_home.admin_order_field = 'home' + + def get_form(self, request, obj=None, **kwargs): + form = super(SystemUserAdmin, self).get_form(request, obj, **kwargs) + form.account = self.account + if obj: + # Has to be done here and not in the form because of strange phenomenon + # derived from monkeypatching formfield.widget.render on AccountAdminMinxin, + # don't ask. + formfield = form.base_fields['groups'] + formfield.queryset = formfield.queryset.exclude(id=obj.id) + return form + + def has_delete_permission(self, request, obj=None): + if obj and obj.is_main: + self.message_user(request, _( + "You have selected one main system user (%(account)s), which can not be deleted.", + ) % {'account': obj}, + messages.ERROR, + ) + + return False + return super(SystemUserAdmin, self).has_delete_permission(request, obj) + + +admin.site.register(SystemUser, SystemUserAdmin) diff --git a/orchestra/contrib/systemusers/api.py b/orchestra/contrib/systemusers/api.py new file mode 100644 index 00000000..b803c810 --- /dev/null +++ b/orchestra/contrib/systemusers/api.py @@ -0,0 +1,23 @@ +from django.utils.translation import gettext_lazy as _ +from rest_framework import viewsets, exceptions + +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from .models import SystemUser +from .serializers import SystemUserSerializer + + +class SystemUserViewSet(LogApiMixin, AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): + queryset = SystemUser.objects.all() + serializer_class = SystemUserSerializer + filter_fields = ('username',) + + def destroy(self, request, pk=None): + user = self.get_object() + if user.is_main: + raise exceptions.PermissionDenied(_("Main system user can not be deleted.")) + return super(SystemUserViewSet, self).destroy(request, pk=pk) + + +router.register(r'systemusers', SystemUserViewSet) diff --git a/orchestra/contrib/systemusers/apps.py b/orchestra/contrib/systemusers/apps.py new file mode 100644 index 00000000..e0db6346 --- /dev/null +++ b/orchestra/contrib/systemusers/apps.py @@ -0,0 +1,28 @@ +import sys + +from django.apps import AppConfig +from django.db.models.signals import post_migrate + +from orchestra.core import services + + +class SystemUsersConfig(AppConfig): + name = 'orchestra.contrib.systemusers' + verbose_name = "System users" + + def ready(self): + from .models import SystemUser + services.register(SystemUser, icon='roleplaying.png') + if 'migrate' in sys.argv and 'accounts' not in sys.argv: + post_migrate.connect(self.create_initial_systemuser, + dispatch_uid="orchestra.contrib.systemusers.apps.create_initial_systemuser") + + def create_initial_systemuser(self, **kwargs): + from .models import SystemUser + Account = SystemUser.account.field.remote_field.model + for account in Account.objects.filter(is_superuser=True, main_systemuser_id__isnull=True): + systemuser = SystemUser.objects.create(username=account.username, + password=account.password, account=account) + account.main_systemuser = systemuser + account.save() + sys.stdout.write("Created initial systemuser %s.\n" % systemuser.username) diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py new file mode 100644 index 00000000..fdc38f25 --- /dev/null +++ b/orchestra/contrib/systemusers/backends.py @@ -0,0 +1,462 @@ +import fnmatch +import os +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace +from orchestra.contrib.resources import ServiceMonitor + +from . import settings + + +class UNIXUserController(ServiceController): + """ + Basic UNIX system user/group support based on useradd, usermod, userdel and groupdel. + Autodetects and uses ACL if available, for better permission management. + """ + verbose_name = _("UNIX user") + model = 'systemusers.SystemUser' + actions = ('save', 'delete', 'set_permission', 'validate_paths_exist', 'create_link') + doc_settings = (settings, ( + 'SYSTEMUSERS_DEFAULT_GROUP_MEMBERS', + 'SYSTEMUSERS_MOVE_ON_DELETE_PATH', + 'SYSTEMUSERS_FORBIDDEN_PATHS' + )) + + def save(self, user): + context = self.get_context(user) + if not context['user']: + return + if not user.active: + self.append(textwrap.dedent(""" + #Just disable that user, if it exists + if id %(user)s ; then + usermod %(user)s --password '%(password)s' + fi + """) % context) + return + # TODO userd add will fail if %(user)s group already exists + self.append(textwrap.dedent(""" + # Update/create user state for %(user)s + if id %(user)s ; then + usermod %(user)s --home '%(home)s' \\ + --password '%(password)s' \\ + --shell '%(shell)s' \\ + --groups '%(groups)s' + else + useradd_code=0 + useradd %(user)s --home '%(home)s' \\ + --password '%(password)s' \\ + --shell '%(shell)s' \\ + --groups '%(groups)s' || useradd_code=$? + if [[ $useradd_code -eq 8 ]]; then + # User is logged in, kill and retry + pkill -u %(user)s; sleep 2 + pkill -9 -u %(user)s; sleep 1 + useradd %(user)s --home '%(home)s' \\ + --password '%(password)s' \\ + --shell '%(shell)s' \\ + --groups '%(groups)s' + elif [[ $useradd_code -ne 0 ]]; then + exit $useradd_code + fi + fi + mkdir -p '%(base_home)s' + chmod 750 '%(base_home)s' + """) % context + ) + if context['home'] != context['base_home']: + self.append(textwrap.dedent("""\ + # Set extra permissions: %(user)s home is inside %(mainuser)s home + if true; then +# if mount | grep "^$(df %(home)s|grep '^/'|cut -d' ' -f1)\s" | grep acl > /dev/null; then + # Account group as the owner + chown %(mainuser)s:%(mainuser)s '%(home)s' + chmod g+s '%(home)s' + # Home access + setfacl -m u:%(user)s:--x '%(mainuser_home)s' + # Grant perms to future files within the directory + setfacl -m d:u:%(user)s:rwx '%(home)s' + # Grant access to main user + setfacl -m d:u:%(mainuser)s:rwx '%(home)s' + else + chmod g+rxw %(home)s + fi""") % context + ) + else: + self.append(textwrap.dedent("""\ + chown %(user)s:%(group)s '%(home)s' + ls -A /etc/skel/ | while read line; do + if [[ ! -e "%(home)s/${line}" ]]; then + cp -a "/etc/skel/${line}" "%(home)s/${line}" && \\ + chown -R %(user)s:%(group)s "%(home)s/${line}" + fi + done + """) % context + ) + for member in settings.SYSTEMUSERS_DEFAULT_GROUP_MEMBERS: + context['member'] = member + self.append('usermod -a -G %(user)s %(member)s || exit_code=$?' % context) + if not user.is_main: + self.append('usermod -a -G %(user)s %(mainuser)s || exit_code=$?' % context) + + def delete(self, user): + context = self.get_context(user) + if not context['user']: + return + self.append(textwrap.dedent(""" + # Delete %(user)s user + nohup bash -c 'sleep 2 && killall -u %(user)s -s KILL' &> /dev/null & + killall -u %(user)s || true + userdel %(user)s || exit_code=$? + groupdel %(group)s || exit_code=$?\ + """) % context + ) + if context['deleted_home']: + self.append(textwrap.dedent("""\ + # Move home into SYSTEMUSERS_MOVE_ON_DELETE_PATH, nesting if exists. + deleted_home="%(deleted_home)s" + while [[ -e "$deleted_home" ]]; do + deleted_home="${deleted_home}/$(basename ${deleted_home})" + done + mv '%(base_home)s' "$deleted_home" || exit_code=$? + """) % context + ) + else: + self.append("rm -fr -- '%(base_home)s'" % context) + + def grant_permissions(self, user, context): + context['perms'] = user.set_perm_perms + # Capital X adds execution permissions for directories, not files + context['perms_X'] = context['perms'] + 'X' + self.append(textwrap.dedent("""\ + # Grant execution permissions to every parent directory + for access_path in %(access_paths)s; do + # Preserve existing ACLs + acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && { + perms=$(echo "$acl" | cut -d':' -f3) + perms=$(echo "$perms" | cut -c 1,2)x + setfacl -m u:%(user)s:$perms "$access_path" + } || setfacl -m u:%(user)s:--x "$access_path" + done + # Grant perms to existing files, excluding execution + find '%(perm_to)s' -type f %(exclude_acl)s \\ + -exec setfacl -m u:%(user)s:%(perms)s {} \\; + # Grant perms to extisting directories and set defaults for future content + find '%(perm_to)s' -type d %(exclude_acl)s \\ + -exec setfacl -m u:%(user)s:%(perms_X)s -m d:u:%(user)s:%(perms_X)s {} \\; + # Account group as the owner of new files + chmod g+s '%(perm_to)s'""") % context + ) + if not user.is_main: + self.append(textwrap.dedent("""\ + # Grant access to main user + find '%(perm_to)s' -type d %(exclude_acl)s \\ + -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\ + """) % context + ) + + def revoke_permissions(self, user, context): + revoke_perms = { + 'rw': '', + 'r': 'w', + 'w': 'r', + } + context.update({ + 'perms': revoke_perms[user.set_perm_perms], + 'option': '-x' if user.set_perm_perms == 'rw' else '-m' + }) + self.append(textwrap.dedent("""\ + # Revoke permissions + find '%(perm_to)s' %(exclude_acl)s \\ + -exec setfacl %(option)s u:%(user)s:%(perms)s {} \\;\ + """) % context + ) + + def set_permission(self, user): + context = self.get_context(user) + context.update({ + 'perm_action': user.set_perm_action, + 'perm_to': os.path.join(user.set_perm_base_home, user.set_perm_home_extension), + }) + exclude_acl = [] + for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS: + context['exclude_acl'] = os.path.join(user.set_perm_base_home, exclude) + exclude_acl.append('-not -path "%(exclude_acl)s"' % context) + context['exclude_acl'] = ' \\\n -a '.join(exclude_acl) if exclude_acl else '' + # Access paths + head = user.set_perm_base_home + relative = '' + access_paths = ["'%s'" % head] + for tail in user.set_perm_home_extension.split(os.sep)[:-1]: + relative = os.path.join(relative, tail) + for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS: + if fnmatch.fnmatch(relative, exclude): + break + else: + # No match + head = os.path.join(head, tail) + access_paths.append("'%s'" % head) + context['access_paths'] = ' '.join(access_paths) + + if user.set_perm_action == 'grant': + self.grant_permissions(user, context) + elif user.set_perm_action == 'revoke': + self.revoke_permissions(user, context) + else: + raise NotImplementedError() + + def create_link(self, user): + context = self.get_context(user) + context.update({ + 'link_target': user.create_link_target, + 'link_name': user.create_link_name, + }) + self.append(textwrap.dedent("""\ + # Create link + su - %(user)s --shell /bin/bash << 'EOF' || exit_code=1 + if [[ ! -e '%(link_name)s' ]]; then + ln -s '%(link_target)s' '%(link_name)s' + else + echo "%(link_name)s already exists, doing nothing." >&2 + exit 1 + fi + EOF""") % context + ) + + def validate_paths_exist(self, user): + for path in user.paths_to_validate: + context = { + 'path': path, + } + self.append(textwrap.dedent(""" + if [[ ! -e '%(path)s' ]]; then + echo "%(path)s path does not exists." >&2 + fi""") % context + ) + + def get_groups(self, user): + if user.is_main: + return user.account.systemusers.exclude(username=user.username).values_list('username', flat=True) + return list(user.groups.values_list('username', flat=True)) + + def get_context(self, user): + context = { + 'object_id': user.pk, + 'user': user.username, + 'group': user.username, + 'groups': ','.join(self.get_groups(user)), + 'password': user.password if user.active else '*%s' % user.password, + 'shell': user.shell, + 'mainuser': user.username if user.is_main else user.account.username, + 'home': user.get_home(), + 'base_home': user.get_base_home(), + 'mainuser_home': user.main.get_home(), + } + context['deleted_home'] = settings.SYSTEMUSERS_MOVE_ON_DELETE_PATH % context + return replace(context, "'", '"') + + +class UNIXUserDisk(ServiceMonitor): + """ + du -bs <home> + """ + model = 'systemusers.SystemUser' + resource = ServiceMonitor.DISK + verbose_name = _('UNIX user disk') + delete_old_equal_values = True + + def prepare(self): + super(UNIXUserDisk, self).prepare() + self.append(textwrap.dedent("""\ + function monitor () { + { SIZE=$(du -bs "$1") && echo $SIZE || echo 0; } | awk {'print $1'} + }""" + )) + + def monitor(self, user): + context = self.get_context(user) + self.append("echo %(object_id)s $(monitor %(base_home)s)" % context) + + def get_context(self, user): + context = { + 'object_id': user.pk, + 'base_home': user.get_base_home(), + } + return replace(context, "'", '"') + + +class Exim4Traffic(ServiceMonitor): + """ + Exim4 mainlog parser for mails sent on the webserver by system users (e.g. via PHP mail()) + """ + model = 'systemusers.SystemUser' + resource = ServiceMonitor.TRAFFIC + verbose_name = _("Exim4 traffic") + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + doc_settings = (settings, + ('SYSTEMUSERS_MAIL_LOG_PATH',) + ) + + def prepare(self): + mainlog = settings.SYSTEMUSERS_MAIL_LOG_PATH + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'mainlogs': str((mainlog, mainlog+'.1')), + } + self.append(textwrap.dedent("""\ + import re + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + mainlogs = {mainlogs} + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + users = {{}} + + def prepare(object_id, username, ini_date): + global users + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + users[username] = [ini_date, object_id, 0] + + def monitor(users, end_date, mainlogs): + user_regex = re.compile(r' U=([^ ]+) ') + for mainlog in mainlogs: + try: + with open(mainlog, 'r') as mainlog: + for line in mainlog.readlines(): + if ' <= ' in line and 'P=local' in line: + username = user_regex.search(line).groups()[0] + try: + sender = users[username] + except KeyError: + continue + else: + date, time, id, __, __, user, protocol, size = line.split()[:8] + date = date.replace('-', '') + date += time.replace(':', '') + if sender[0] < int(date) < end_date: + sender[2] += int(size[2:]) + except IOError as e: + sys.stderr.write(str(e)) + + for username, opts in users.iteritems(): + __, object_id, size = opts + print object_id, size + """).format(**context) + ) + + def commit(self): + self.append('monitor(users, end_date, mainlogs)') + + def monitor(self, user): + context = self.get_context(user) + self.append("prepare(%(object_id)s, '%(username)s', '%(last_date)s')" % context) + + def get_context(self, user): + context = { + 'username': user.username, + 'object_id': user.pk, + 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + } + return context + + +class VsFTPdTraffic(ServiceMonitor): + """ + vsFTPd log parser. + """ + model = 'systemusers.SystemUser' + resource = ServiceMonitor.TRAFFIC + verbose_name = _('VsFTPd traffic') + script_executable = '/usr/bin/python' + monthly_sum_old_values = True + doc_settings = (settings, + ('SYSTEMUSERS_FTP_LOG_PATH',) + ) + + def prepare(self): + vsftplog = settings.SYSTEMUSERS_FTP_LOG_PATH + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'vsftplogs': str((vsftplog, vsftplog+'.1')), + } + self.append(textwrap.dedent("""\ + import re + import sys + from datetime import datetime + from dateutil import tz + + def to_local_timezone(date, tzlocal=tz.tzlocal()): + date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') + date = date.replace(tzinfo=tz.tzutc()) + date = date.astimezone(tzlocal) + return date + + vsftplogs = {vsftplogs} + # Use local timezone + end_date = to_local_timezone('{current_date}') + end_date = int(end_date.strftime('%Y%m%d%H%M%S')) + users = {{}} + months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') + months = dict((m, '%02d' % n) for n, m in enumerate(months, 1)) + + def prepare(object_id, username, ini_date): + global users + ini_date = to_local_timezone(ini_date) + ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) + users[username] = [ini_date, object_id, 0] + + def monitor(users, end_date, months, vsftplogs): + user_regex = re.compile(r'\] \[([^ ]+)\] (OK|FAIL) ') + bytes_regex = re.compile(r', ([0-9]+) bytes, ') + for vsftplog in vsftplogs: + try: + with open(vsftplog, 'r') as vsftplog: + for line in vsftplog.readlines(): + if ' bytes, ' in line: + username = user_regex.search(line).groups()[0] + try: + user = users[username] + except KeyError: + continue + else: + __, month, day, time, year = line.split()[:5] + date = year + months[month] + day + time.replace(':', '') + if user[0] < int(date) < end_date: + bytes = bytes_regex.search(line).groups()[0] + user[2] += int(bytes) + except IOError as e: + sys.stderr.write(str(e)) + + for username, opts in users.items(): + __, object_id, size = opts + print object_id, size + """).format(**context) + ) + + def monitor(self, user): + context = self.get_context(user) + self.append("prepare(%(object_id)s, '%(username)s', '%(last_date)s')" % context) + + def commit(self): + self.append('monitor(users, end_date, months, vsftplogs)') + + def get_context(self, user): + context = { + 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'object_id': user.pk, + 'username': user.username, + } + return replace(context, "'", '"') diff --git a/orchestra/contrib/systemusers/filters.py b/orchestra/contrib/systemusers/filters.py new file mode 100644 index 00000000..7d1d9728 --- /dev/null +++ b/orchestra/contrib/systemusers/filters.py @@ -0,0 +1,20 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + + +class IsMainListFilter(SimpleListFilter): + """ Filter Nodes by group according to request.user """ + title = _("main") + parameter_name = 'is_main' + + def lookups(self, request, model_admin): + return ( + ('True', _("Yes")), + ('False', _("No")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.by_is_main() + if self.value() == 'False': + return queryset.by_is_main(is_main=False) diff --git a/orchestra/contrib/systemusers/forms.py b/orchestra/contrib/systemusers/forms.py new file mode 100644 index 00000000..f95529bb --- /dev/null +++ b/orchestra/contrib/systemusers/forms.py @@ -0,0 +1,164 @@ +import os +import textwrap + +from django import forms +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra.forms import UserCreationForm, UserChangeForm + +from . import settings +from .models import SystemUser +from .validators import validate_home, validate_paths_exist + + +class SystemUserFormMixin(object): + MOCK_USERNAME = '' + + def __init__(self, *args, **kwargs): + super(SystemUserFormMixin, self).__init__(*args, **kwargs) + duplicate = lambda n: (n, n) + if self.instance.pk: + username = self.instance.username + choices=( + duplicate(self.account.main_systemuser.get_base_home()), + duplicate(self.instance.get_base_home()), + ) + else: + username = self.MOCK_USERNAME + choices=( + duplicate(self.account.main_systemuser.get_base_home()), + duplicate(SystemUser(username=username).get_base_home()), + ) + self.fields['home'].widget = forms.Select(choices=choices) + if self.instance.pk and (self.instance.is_main or self.instance.has_shell): + # hidde home option for shell users + self.fields['home'].widget.input_type = 'hidden' + self.fields['directory'].widget.input_type = 'hidden' + elif self.instance.pk and (self.instance.get_base_home() == self.instance.home): + self.fields['directory'].widget = forms.HiddenInput() + else: + self.fields['directory'].widget = forms.TextInput(attrs={'size':'70'}) + if not self.instance.pk or not self.instance.is_main: + # Some javascript for hidde home/directory inputs when convinient + self.fields['shell'].widget.attrs['onChange'] = textwrap.dedent("""\ + field = $(".field-home, .field-directory"); + input = $("#id_home, #id_directory"); + if ($.inArray(this.value, %s) < 0) { + field.addClass("hidden"); + } else { + field.removeClass("hidden"); + input.removeAttr("type"); + };""" % list(settings.SYSTEMUSERS_DISABLED_SHELLS) + ) + self.fields['home'].widget.attrs['onChange'] = textwrap.dedent("""\ + field = $(".field-box.field-directory"); + input = $("#id_directory"); + if (this.value.search("%s") > 0) { + field.addClass("hidden"); + } else { + field.removeClass("hidden"); + input.removeAttr("type"); + };""" % username + ) + + def clean_directory(self): + directory = self.cleaned_data['directory'] + return directory.lstrip('/') + + def clean(self): + super(SystemUserFormMixin, self).clean() + cleaned_data = self.cleaned_data + home = cleaned_data.get('home') + shell = cleaned_data.get('shell') + if home and self.MOCK_USERNAME in home: + username = cleaned_data.get('username', '') + cleaned_data['home'] = home.replace(self.MOCK_USERNAME, username) + elif home and shell not in settings.SYSTEMUSERS_DISABLED_SHELLS: + cleaned_data['home'] = '' + cleaned_data['directory'] = '' + validate_home(self.instance, cleaned_data, self.account) + return cleaned_data + + +class SystemUserCreationForm(SystemUserFormMixin, UserCreationForm): + pass + + +class SystemUserChangeForm(SystemUserFormMixin, UserChangeForm): + pass + + +class LinkForm(forms.Form): + base_home = forms.ChoiceField(label=_("Target path"), choices=(), + help_text=_("Target link will be under this directory.")) + home_extension = forms.CharField(label=_("Home extension"), required=False, initial='', + widget=forms.TextInput(attrs={'size':'70'}), + help_text=_("Relative path to chosen directory.")) + link_name = forms.CharField(label=_("Link name"), required=False, initial='', + widget=forms.TextInput(attrs={'size':'70'})) + + def __init__(self, *args, **kwargs): + self.instance = args[0] + self.queryset = kwargs.pop('queryset', []) + super_args = [] + if len(args) > 1: + super_args.append(args[1]) + super(LinkForm, self).__init__(*super_args, **kwargs) + related_users = type(self.instance).objects.filter(account=self.instance.account_id) + self.fields['base_home'].choices = ( + (user.get_base_home(), user.get_base_home()) for user in related_users + ) + if len(self.queryset) == 1: + user = self.instance + help_text = _("If left blank or relative path: the link will be created in %s home.") % user + else: + help_text = _("If left blank or relative path: the link will be created in each user home.") + self.fields['link_name'].help_text = help_text + + def clean_home_extension(self): + home_extension = self.cleaned_data['home_extension'] + return home_extension.lstrip('/') + + def clean_link_name(self): + link_name = self.cleaned_data['link_name'] + if link_name: + if link_name.startswith('/'): + if len(self.queryset) > 1: + raise ValidationError( + _("Link name can not be a full path when multiple users.")) + link_names = [os.path.dirname(link_name)] + else: + dir_name = os.path.dirname(link_name) + link_names = [os.path.join(user.home, dir_name) for user in self.queryset] + validate_paths_exist(self.instance, link_names) + return link_name + + def clean(self): + cleaned_data = super(LinkForm, self).clean() + path = os.path.join(cleaned_data['base_home'], cleaned_data['home_extension']) + try: + validate_paths_exist(self.instance, [path]) + except ValidationError as err: + raise ValidationError({ + 'home_extension': err, + }) + return cleaned_data + + +class PermissionForm(LinkForm): + set_action = forms.ChoiceField(label=_("Action"), initial='grant', + choices=( + ('grant', _("Grant")), + ('revoke', _("Revoke")) + )) + base_home = forms.ChoiceField(label=_("Set permissions to"), choices=(), + help_text=_("User will be granted/revoked access to this directory.")) + home_extension = forms.CharField(label=_("Home extension"), required=False, initial='', + widget=forms.TextInput(attrs={'size':'70'}), help_text=_("Relative to chosen home.")) + permissions = forms.ChoiceField(label=_("Permissions"), initial='read-write', + choices=( + ('rw', _("Read and write")), + ('r', _("Read only")), + ('w', _("Write only")) + )) diff --git a/orchestra/contrib/systemusers/models.py b/orchestra/contrib/systemusers/models.py new file mode 100644 index 00000000..bd6dc048 --- /dev/null +++ b/orchestra/contrib/systemusers/models.py @@ -0,0 +1,138 @@ +import fnmatch +import os + +from django.contrib.auth.hashers import make_password +from django.core.exceptions import ValidationError +from django.db import models +from django.db.models import F +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import validators + +from . import settings + + +class SystemUserQuerySet(models.QuerySet): + def create_user(self, username, password='', **kwargs): + user = super(SystemUserQuerySet, self).create(username=username, **kwargs) + user.set_password(password) + user.save(update_fields=['password']) + return user + + def by_is_main(self, is_main=True, **kwargs): + if is_main: + return self.filter(account__main_systemuser_id=F('id')) + else: + return self.exclude(account__main_systemuser_id=F('id')) + + +class SystemUser(models.Model): + """ + System users + + Username max_length determined by LINUX system user/group lentgh: 32 + """ + username = models.CharField(_("username"), max_length=32, unique=True, + help_text=_("Required. 32 characters or fewer. Letters, digits and ./-/_ only."), + validators=[validators.validate_username]) + password = models.CharField(_("password"), max_length=128) + account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), + related_name='systemusers', on_delete=models.CASCADE) + home = models.CharField(_("home"), max_length=256, blank=True, + help_text=_("Starting location when login with this no-shell user.")) + directory = models.CharField(_("directory"), max_length=256, blank=True, + help_text=_("Optional directory relative to user's home.")) + shell = models.CharField(_("shell"), max_length=32, choices=settings.SYSTEMUSERS_SHELLS, + default=settings.SYSTEMUSERS_DEFAULT_SHELL) + groups = models.ManyToManyField('self', blank=True, symmetrical=False, + help_text=_("A new group will be created for the user. " + "Which additional groups would you like them to be a member of?")) + is_active = models.BooleanField(_("active"), default=True, + help_text=_("Designates whether this account should be treated as active. " + "Unselect this instead of deleting accounts.")) + + objects = SystemUserQuerySet.as_manager() + + def __str__(self): + return self.username + + @cached_property + def active(self): + try: + return self.is_active and self.account.is_active + except type(self).account.field.related_model.DoesNotExist: + return self.is_active + + @cached_property + def is_main(self): + # TODO on account delete + # On account creation main_systemuser_id is still None + if self.account.main_systemuser_id: + return self.account.main_systemuser_id == self.pk + return self.account.username == self.username + + @cached_property + def main(self): + # On account creation main_systemuser_id is still None + if self.account.main_systemuser_id: + return self.account.main_systemuser + return type(self).objects.get(username=self.account.username) + + @property + def has_shell(self): + return self.shell not in settings.SYSTEMUSERS_DISABLED_SHELLS + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = True + self.save(update_fields=('is_active',)) + + def get_description(self): + return self.get_shell_display() + + def save(self, *args, **kwargs): + if not self.home: + self.home = self.get_base_home() + super(SystemUser, self).save(*args, **kwargs) + + def clean(self): + self.directory = self.directory.lstrip('/') + if self.home: + self.home = os.path.normpath(self.home) + if self.directory: + self.directory = os.path.normpath(self.directory) + dir_errors = [] + if self.has_shell: + dir_errors.append(_("Directory with shell users can not be specified.")) + elif self.account_id and self.is_main: + dir_errors.append(_("Directory with main system users can not be specified.")) + elif self.home == self.get_base_home(): + dir_errors.append(_("Directory on the user's base home is not allowed.")) + for pattern in settings.SYSTEMUSERS_FORBIDDEN_PATHS: + if fnmatch.fnmatch(self.directory, pattern): + dir_errors.append(_("Provided directory is forbidden.")) + if dir_errors: + raise ValidationError({ + 'directory': [ValidationError(error) for error in dir_errors] + }) + if self.has_shell and self.home and self.home != self.get_base_home(): + raise ValidationError({ + 'home': _("Shell users should use their own home."), + }) + + def set_password(self, raw_password): + self.password = make_password(raw_password) + + def get_base_home(self): + context = { + 'user': self.username, + 'username': self.username, + } + return os.path.normpath(settings.SYSTEMUSERS_HOME % context) + + def get_home(self): + return os.path.normpath(os.path.join(self.home, self.directory)) diff --git a/orchestra/contrib/systemusers/serializers.py b/orchestra/contrib/systemusers/serializers.py new file mode 100644 index 00000000..5083b68e --- /dev/null +++ b/orchestra/contrib/systemusers/serializers.py @@ -0,0 +1,43 @@ +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.api.serializers import SetPasswordHyperlinkedSerializer, RelatedHyperlinkedModelSerializer +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .models import SystemUser +from .validators import validate_home + + +class RelatedGroupSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = SystemUser + fields = ('url', 'id', 'username',) + + +class SystemUserSerializer(AccountSerializerMixin, SetPasswordHyperlinkedSerializer): + groups = RelatedGroupSerializer(many=True, required=False) + + class Meta: + model = SystemUser + fields = ( + 'url', 'id', 'username', 'password', 'home', 'directory', 'shell', 'groups', 'is_active', + ) + postonly_fields = ('username', 'password') + + def validate_directory(self, directory): + return directory.lstrip('/') + + def validate(self, data): + data = super(SystemUserSerializer, self).validate(data) + user = SystemUser( + username=data.get('username') or self.instance.username, + shell=data.get('shell') or self.instance.shell, + ) + validate_home(user, data, self.get_account()) + groups = data.get('groups') + if groups: + for group in groups: + if group.username == data['username']: + raise serializers.ValidationError( + _("Do not make the user member of its group")) + return data diff --git a/orchestra/contrib/systemusers/settings.py b/orchestra/contrib/systemusers/settings.py new file mode 100644 index 00000000..5e931a02 --- /dev/null +++ b/orchestra/contrib/systemusers/settings.py @@ -0,0 +1,66 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting + + +_names = ('user', 'username') +_backend_names = _names + ('group', 'shell', 'mainuser', 'home', 'base_home') + + +SYSTEMUSERS_SHELLS = Setting('SYSTEMUSERS_SHELLS', + ( + ('/dev/null', _("No shell, FTP only")), + ('/bin/rssh', _("No shell, SFTP/RSYNC only")), + ('/bin/bash', "/bin/bash"), + ), + validators=[Setting.validate_choices] +) + + +SYSTEMUSERS_DEFAULT_SHELL = Setting('SYSTEMUSERS_DEFAULT_SHELL', + '/dev/null', + choices=SYSTEMUSERS_SHELLS +) + + +SYSTEMUSERS_DISABLED_SHELLS = Setting('SYSTEMUSERS_DISABLED_SHELLS', + default=( + '/dev/null', + '/bin/rssh', + ), +) + + +SYSTEMUSERS_HOME = Setting('SYSTEMUSERS_HOME', + '/home/%(user)s', + help_text="Available fromat names: %s" % ', '.join(_names), + validators=[Setting.string_format_validator(_names)], +) + + +SYSTEMUSERS_FTP_LOG_PATH = Setting('SYSTEMUSERS_FTP_LOG_PATH', + '/var/log/vsftpd.log' +) + + +SYSTEMUSERS_MAIL_LOG_PATH = Setting('SYSTEMUSERS_MAIL_LOG_PATH', + '/var/log/exim4/mainlog' +) + +SYSTEMUSERS_DEFAULT_GROUP_MEMBERS = Setting('SYSTEMUSERS_DEFAULT_GROUP_MEMBERS', + ('www-data',) +) + + +SYSTEMUSERS_MOVE_ON_DELETE_PATH = Setting('SYSTEMUSERS_MOVE_ON_DELETE_PATH', + '', + help_text="Available fromat names: %s" % ', '.join(_backend_names), + validators=[Setting.string_format_validator(_backend_names)], +) + + +SYSTEMUSERS_FORBIDDEN_PATHS = Setting('SYSTEMUSERS_FORBIDDEN_PATHS', + (), + help_text=("Exlude ACL operations or home locations on provided globs, relative to user's home.
    " + "e.g. ('logs', 'logs/apache*', 'webapps')"), +) diff --git a/orchestra/contrib/systemusers/templates/admin/systemusers/systemuser/create_link.html b/orchestra/contrib/systemusers/templates/admin/systemusers/systemuser/create_link.html new file mode 100644 index 00000000..d55af694 --- /dev/null +++ b/orchestra/contrib/systemusers/templates/admin/systemusers/systemuser/create_link.html @@ -0,0 +1,73 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n %} +{% load admin_urls static utils %} + +{% block extrastyle %} +{{ block.super }} + + +{% endblock %} + +{% block breadcrumbs %} +
    +{% trans 'Home' %} +› {{ app_label|capfirst|escape }} +› {{ opts.verbose_name_plural|capfirst }} +{% if obj %} +› {{ obj }} +› {{ action_name }} +{% elif add %} +› {% trans "Add" %} {{ opts.verbose_name }} +› {{ action_name }} +{% else %} +› {{ action_name }} multiple objects +{% endif %} +
    +{% endblock %} + +{% block content %} +
    +
    + {% block introduction %} + Create simbolic link for {% for user in queryset %}{{ user.username }}{% if not forloop.last %}, {% endif %}{% endfor %}. + {% endblock %} +
      {{ display_objects | unordered_list }}
    +
    {% csrf_token %} +
    + {{ form.non_field_errors }} + {% block prefields %} + {% endblock %} +
    +
    + {{ form.base_home.errors }} + + {{ form.base_home }}{% for x in ""|ljust:"50" %} {% endfor %} +

    {{ form.base_home.help_text|safe }}

    +
    +
    + {{ form.home_extension.errors }} + + {{ form.home_extension }} +

    {{ form.home_extension.help_text|safe }}

    +
    +
    + {% block postfields %} +
    + {{ form.link_name.errors }} + + {{ form.link_name }} +

    {{ form.link_name.help_text|safe }}

    +
    + {% endblock %} +
    +
    + {% for obj in queryset %} + + {% endfor %} + + + +
    +
    +{% endblock %} + diff --git a/orchestra/contrib/systemusers/templates/admin/systemusers/systemuser/set_permission.html b/orchestra/contrib/systemusers/templates/admin/systemusers/systemuser/set_permission.html new file mode 100644 index 00000000..6d051251 --- /dev/null +++ b/orchestra/contrib/systemusers/templates/admin/systemusers/systemuser/set_permission.html @@ -0,0 +1,26 @@ +{% extends "admin/systemusers/systemuser/create_link.html" %} +{% load i18n l10n %} +{% load admin_urls static utils %} + +{% block introduction %} +Set permissions for {% for user in queryset %}{{ user.username }}{% if not forloop.last %}, {% endif %}{% endfor %} system user(s). +{% endblock %} + + +{% block prefields %} +
    + {{ form.set_action.errors }} + + {{ form.set_action }}{% for x in ""|ljust:"50" %} {% endfor %} +

    {{ form.set_action.help_text|safe }}

    +
    +{% endblock %} + +{% block postfields %} +
    + {{ form.permissions.errors }} + + {{ form.permissions }}{% for x in ""|ljust:"50" %} {% endfor %} +

    {{ form.permissions.help_text|safe }}

    +
    +{% endblock %} diff --git a/orchestra/contrib/systemusers/tests/__init__.py b/orchestra/contrib/systemusers/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/systemusers/tests/functional_tests/__init__.py b/orchestra/contrib/systemusers/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/systemusers/tests/functional_tests/tests.py b/orchestra/contrib/systemusers/tests/functional_tests/tests.py new file mode 100644 index 00000000..9d3cf2e0 --- /dev/null +++ b/orchestra/contrib/systemusers/tests/functional_tests/tests.py @@ -0,0 +1,378 @@ +import ftplib +import os +import re +import time +import unittest +from functools import partial + +import paramiko +from django.conf import settings as djsettings +from django.core.management.base import CommandError +from django.urls import reverse +from selenium.webdriver.support.select import Select + +from orchestra.admin.utils import change_url +from orchestra.contrib.accounts.models import Account +from orchestra.contrib.orchestration.models import Server, Route +from orchestra.utils.sys import run, sshrun +from orchestra.utils.tests import (BaseLiveServerTestCase, random_ascii, snapshot_on_error, + save_response_on_error) + +from ... import backends +from ...models import SystemUser + + +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) +r = partial(run, silent=True, display=False) +sshr = partial(sshrun, silent=True, display=False) + + +class SystemUserMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_MASTER_SERVER', 'localhost') + DEPENDENCIES = ( + 'orchestra.contrib.orchestration', + 'orcgestra.apps.systemusers', + ) + + def setUp(self): + super(SystemUserMixin, self).setUp() + self.add_route() + djsettings.DEBUG = True + + def add_route(self): + master = Server.objects.create(name=self.MASTER_SERVER) + backend = backends.UNIXUserController.get_name() + Route.objects.create(backend=backend, match=True, host=master) + + def save(self): + raise NotImplementedError + + def add(self): + raise NotImplementedError + + def delete(self): + raise NotImplementedError + + def update(self): + raise NotImplementedError + + def disable(self): + raise NotImplementedError + + def add_group(self, username, groupname): + raise NotImplementedError + + def validate_user(self, username): + idcmd = sshr(self.MASTER_SERVER, "id %s" % username) + self.assertEqual(0, idcmd.exit_code) + user = SystemUser.objects.get(username=username) + groups = list(user.groups.values_list('username', flat=True)) + groups.append(user.username) + idgroups = idcmd.stdout.strip().split(' ')[2] + idgroups = re.findall(r'\d+\((\w+)\)', idgroups) + self.assertEqual(set(groups), set(idgroups)) + + def validate_delete(self, username): + self.assertRaises(SystemUser.DoesNotExist, SystemUser.objects.get, username=username) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'id %s' % username, display=False) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'grep "^%s:" /etc/groups' % username, display=False) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'grep "^%s:" /etc/passwd' % username, display=False) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'grep "^%s:" /etc/shadow' % username, display=False) + # Home will be deleted on account delete, see test_delete_account + + def validate_ftp(self, username, password): + ftp = ftplib.FTP(self.MASTER_SERVER) + ftp.login(user=username, passwd=password) + ftp.close() + + def validate_sftp(self, username, password): + transport = paramiko.Transport((self.MASTER_SERVER, 22)) + transport.connect(username=username, password=password) + sftp = paramiko.SFTPClient.from_transport(transport) + sftp.listdir() + sftp.close() + + def validate_ssh(self, username, password): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(self.MASTER_SERVER, username=username, password=password) + transport = ssh.get_transport() + channel = transport.open_session() + channel.exec_command('ls') + self.assertEqual(0, channel.recv_exit_status()) + channel.close() + + def test_add(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + self.validate_user(username) + + def test_ftp(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password, shell='/dev/null') + self.addCleanup(self.delete, username) + self.assertRaises(paramiko.AuthenticationException, + self.validate_sftp, username, password) + self.assertRaises(paramiko.AuthenticationException, + self.validate_ssh, username, password) + + def test_sftp(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password, shell='/bin/rssh') + self.addCleanup(self.delete, username) + self.validate_sftp(username, password) + self.assertRaises(AssertionError, self.validate_ssh, username, password) + + def test_ssh(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password, shell='/bin/bash') + self.addCleanup(self.delete, username) + self.validate_ssh(username, password) + + def test_delete(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%sppppP001' % random_ascii(5) + self.add(username, password) + self.validate_user(username) + self.delete(username) + self.validate_delete(username) + self.assertRaises(Exception, self.delete, self.account.username) + + def test_add_group(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + self.validate_user(username) + username2 = '%s_systemuser' % random_ascii(10) + password2 = '@!?%spppP001' % random_ascii(5) + self.add(username2, password2) + self.addCleanup(self.delete, username2) + self.validate_user(username2) + self.add_group(username, username2) + user = SystemUser.objects.get(username=username) + groups = list(user.groups.values_list('username', flat=True)) + self.assertIn(username2, groups) + self.validate_user(username) + + def test_disable(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password, shell='/dev/null') + self.addCleanup(self.delete, username) + self.validate_ftp(username, password) + self.disable(username) + self.validate_user(username) + self.assertRaises(ftplib.error_perm, self.validate_ftp, username, password) + + def test_change_password(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + self.validate_ftp(username, password) + new_password = '@!?%spppP001' % random_ascii(5) + self.change_password(username, new_password) + self.validate_ftp(username, new_password) + +# TODO test resources + + +@unittest.skipUnless(TEST_REST_API, "REST API tests") +class RESTSystemUserMixin(SystemUserMixin): + def setUp(self): + super(RESTSystemUserMixin, self).setUp() + self.rest_login() + # create main user + self.save(self.account.username) + self.addCleanup(self.delete_account, self.account.username) + + @save_response_on_error + def add(self, username, password, shell='/dev/null'): + self.rest.systemusers.create(username=username, password=password, shell=shell) + + @save_response_on_error + def delete(self, username): + user = self.rest.systemusers.retrieve(username=username).get() + user.delete() + + @save_response_on_error + def add_group(self, username, groupname): + user = self.rest.systemusers.retrieve(username=username).get() + user.groups.append({'username': groupname}) + user.save() + + @save_response_on_error + def disable(self, username): + user = self.rest.systemusers.retrieve(username=username).get() + user.is_active = False + user.save() + + @save_response_on_error + def save(self, username): + user = self.rest.systemusers.retrieve(username=username).get() + user.save() + + @save_response_on_error + def change_password(self, username, password): + user = self.rest.systemusers.retrieve(username=username).get() + user.set_password(password) + + def delete_account(self, username): + self.rest.account.delete() + + +class AdminSystemUserMixin(SystemUserMixin): + def setUp(self): + super(AdminSystemUserMixin, self).setUp() + self.admin_login() + # create main user + self.save(self.account.username) + self.addCleanup(self.delete_account, self.account.username) + + @snapshot_on_error + def add(self, username, password, shell='/dev/null'): + url = self.live_server_url + reverse('admin:systemusers_systemuser_add') + self.selenium.get(url) + + username_field = self.selenium.find_element_by_id('id_username') + username_field.send_keys(username) + + password_field = self.selenium.find_element_by_id('id_password1') + password_field.send_keys(password) + password_field = self.selenium.find_element_by_id('id_password2') + password_field.send_keys(password) + + shell_input = self.selenium.find_element_by_id('id_shell') + shell_select = Select(shell_input) + shell_select.select_by_value(shell) + + username_field.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def delete(self, username): + user = SystemUser.objects.get(username=username) + self.admin_delete(user) + + @snapshot_on_error + def delete_account(self, username): + account = Account.objects.get(username=username) + self.admin_delete(account) + + @snapshot_on_error + def disable(self, username): + user = SystemUser.objects.get(username=username) + self.admin_disable(user) + + @snapshot_on_error + def add_group(self, username, groupname): + user = SystemUser.objects.get(username=username) + url = self.live_server_url + change_url(user) + self.selenium.get(url) + groups = self.selenium.find_element_by_id('id_groups_add_all_link') + groups.click() + time.sleep(0.5) + save = self.selenium.find_element_by_name('_save') + save.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def save(self, username): + user = SystemUser.objects.get(username=username) + url = self.live_server_url + change_url(user) + self.selenium.get(url) + save = self.selenium.find_element_by_name('_save') + save.submit() + self.assertNotEqual(url, self.selenium.current_url) + + @snapshot_on_error + def change_password(self, username, password): + user = SystemUser.objects.get(username=username) + self.admin_change_password(user, password) + + +class RESTSystemUserTest(RESTSystemUserMixin, BaseLiveServerTestCase): + pass + + +class AdminSystemUserTest(AdminSystemUserMixin, BaseLiveServerTestCase): + @snapshot_on_error + def test_create_account(self): + url = self.live_server_url + reverse('admin:accounts_account_add') + self.selenium.get(url) + + account_username = '%s_account' % random_ascii(10) + username = self.selenium.find_element_by_id('id_username') + username.send_keys(account_username) + + account_password = '@!?%spppP001' % random_ascii(5) + password = self.selenium.find_element_by_id('id_password1') + password.send_keys(account_password) + password = self.selenium.find_element_by_id('id_password2') + password.send_keys(account_password) + + full_name = random_ascii(10) + full_name_field = self.selenium.find_element_by_id('id_full_name') + full_name_field.send_keys(full_name) + + account_email = 'orchestra@orchestra.lan' + email = self.selenium.find_element_by_id('id_email') + email.send_keys(account_email) + + contact_short_name = random_ascii(10) + short_name = self.selenium.find_element_by_id('id_contacts-0-short_name') + short_name.send_keys(contact_short_name) + + email = self.selenium.find_element_by_id('id_contacts-0-email') + email.send_keys(account_email) + email.submit() + self.assertNotEqual(url, self.selenium.current_url) + + self.addCleanup(self.delete_account, account_username) + self.assertEqual(0, sshr(self.MASTER_SERVER, "id %s" % account_username).exit_code) + + @snapshot_on_error + def test_delete_account(self): + home = self.account.main_systemuser.get_home() + self.admin_delete(self.account) + self.assertRaises(CommandError, run, 'ls %s' % home, display=False) + # Recreate a fucking fake account for test cleanup + self.account = self.create_account(username=self.account.username, superuser=True) + self.selenium.delete_all_cookies() + self.admin_login() + + @snapshot_on_error + def test_disable_account(self): + username = '%s_systemuser' % random_ascii(10) + password = '@!?%spppP001' % random_ascii(5) + self.add(username, password) + self.addCleanup(self.delete, username) + self.validate_ftp(username, password) + self.disable(username) + self.validate_user(username) + + disable = reverse('admin:accounts_account_disable', args=(self.account.pk,)) + url = self.live_server_url + disable + self.selenium.get(url) + confirmation = self.selenium.find_element_by_name('post') + confirmation.submit() + self.assertNotEqual(url, self.selenium.current_url) + + self.assertRaises(ftplib.error_perm, self.validate_ftp, username, password) + self.selenium.get(url) + self.assertNotEqual(url, self.selenium.current_url) + + # Reenable for test cleanup + self.account.is_active = True + self.account.save() + self.admin_login() diff --git a/orchestra/contrib/systemusers/validators.py b/orchestra/contrib/systemusers/validators.py new file mode 100644 index 00000000..cafea1c2 --- /dev/null +++ b/orchestra/contrib/systemusers/validators.py @@ -0,0 +1,48 @@ +import os + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import Operation + + +def validate_paths_exist(user, paths): + operations = [] + user.paths_to_validate = paths + operations.extend(Operation.create_for_action(user, 'validate_paths_exist')) + logs = Operation.execute(operations) + stderr = '\n'.join([log.stderr for log in logs]) + if 'path does not exists' in stderr: + raise ValidationError(stderr) + + +def validate_home(user, data, account): + """ validates home based on account and data['shell'] """ + if not 'username' in data and not user.pk: + # other validation will have been raised for required username + return + user = type(user)( + username=data.get('username') or user.username, + shell=data.get('shell') or user.shell, + ) + if 'home' in data and data['home']: + home = os.path.normpath(data['home']) + user_home = user.get_base_home() + account_home = account.main_systemuser.get_home() + if user.has_shell: + if home != user_home: + raise ValidationError({ + 'home': _("Not a valid home directory.") + }) + elif home not in (user_home, account_home): + raise ValidationError({ + 'home': _("Not a valid home directory.") + }) + if 'directory' in data and data['directory']: + path = os.path.join(data['home'], data['directory']) + try: + validate_paths_exist(user, (path,)) + except ValidationError as err: + raise ValidationError({ + 'directory': err, + }) diff --git a/orchestra/contrib/tasks/README.md b/orchestra/contrib/tasks/README.md new file mode 100644 index 00000000..f64f6787 --- /dev/null +++ b/orchestra/contrib/tasks/README.md @@ -0,0 +1,6 @@ +This is a wrapper around djcelery and celery `@task` and `@periodic_task` decorators. It provides transparent support for switching between executing a task on a plain Python thread or +the traditional way of pushing the task on a queue (rabbitmq) and wait for a Celery worker to run it. + +A queueless threaded execution has the advantage of 0 moving parts instead of the alternative rabbitmq and celery workers. Less dependencies, less memory footprint, less points of failure, no process keeping, no independent code reloading for the workers. + +If your application needs to run thousands or milions of tasks a day, use celery as your backend, if tens or hundreds, then probably the default thread backend will be your best choice. diff --git a/orchestra/contrib/tasks/__init__.py b/orchestra/contrib/tasks/__init__.py new file mode 100644 index 00000000..61023b65 --- /dev/null +++ b/orchestra/contrib/tasks/__init__.py @@ -0,0 +1,5 @@ +from . import settings +from .decorators import task, periodic_task, keep_state, apply_async + + +default_app_config = 'orchestra.contrib.tasks.apps.TasksConfig' diff --git a/orchestra/contrib/tasks/admin.py b/orchestra/contrib/tasks/admin.py new file mode 100644 index 00000000..d245a5f6 --- /dev/null +++ b/orchestra/contrib/tasks/admin.py @@ -0,0 +1,10 @@ +from django.utils.translation import gettext_lazy as _ +from djcelery.admin import PeriodicTaskAdmin + +from orchestra.admin.utils import admin_date + + +display_last_run_at = admin_date('last_run_at', short_description=_("Last run")) + + +PeriodicTaskAdmin.list_display = ('__unicode__', display_last_run_at, 'total_run_count', 'enabled') diff --git a/orchestra/contrib/tasks/apps.py b/orchestra/contrib/tasks/apps.py new file mode 100644 index 00000000..ceb4e24b --- /dev/null +++ b/orchestra/contrib/tasks/apps.py @@ -0,0 +1,16 @@ +from django.apps import AppConfig +from django.utils.module_loading import autodiscover_modules + +from orchestra.core import administration + + +class TasksConfig(AppConfig): + name = 'orchestra.contrib.tasks' + verbose_name = "Tasks" + + def ready(self): + from djcelery.models import PeriodicTask, TaskState, WorkerState + administration.register(TaskState, icon='Edit-check-sheet.png') + administration.register(PeriodicTask, parent=TaskState, icon='Appointment.png') + administration.register(WorkerState, parent=TaskState, dashboard=False) + autodiscover_modules('tasks') diff --git a/orchestra/contrib/tasks/beat.py b/orchestra/contrib/tasks/beat.py new file mode 100644 index 00000000..7a4772ac --- /dev/null +++ b/orchestra/contrib/tasks/beat.py @@ -0,0 +1,43 @@ +import json + +from celery import current_app +from celery.schedules import crontab_parser as CrontabParser +from django.utils import timezone +from djcelery.models import PeriodicTask + +from .decorators import apply_async + + +def is_due(task, time=None): + if time is None: + time = timezone.now() + crontab = task.crontab + parts = map(int, time.strftime("%M %H %w %d %m").split()) + n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = parts + return bool( + n_minute in CrontabParser(60).parse(crontab.minute) and + n_hour in CrontabParser(24).parse(crontab.hour) and + n_day_of_week in CrontabParser(7).parse(crontab.day_of_week) and + n_day_of_month in CrontabParser(31, 1).parse(crontab.day_of_month) and + n_month_of_year in CrontabParser(12, 1).parse(crontab.month_of_year) + ) + + +def run_task(task, thread=True, process=False, run_async=False): + args = json.loads(task.args) + kwargs = json.loads(task.kwargs) + task_fn = current_app.tasks.get(task.task) + if run_async: + method = 'process' if process else 'thread' + return apply_async(task_fn, method=method).apply_async(*args, **kwargs) + return task_fn(*args, **kwargs) + + +def run(): + now = timezone.now() + procs = [] + for task in PeriodicTask.objects.enabled().select_related('crontab'): + if is_due(task, now): + proc = run_task(task, process=True, run_async=True) + procs.append(proc) + [proc.join() for proc in procs] diff --git a/orchestra/contrib/tasks/decorators.py b/orchestra/contrib/tasks/decorators.py new file mode 100644 index 00000000..72fd2af7 --- /dev/null +++ b/orchestra/contrib/tasks/decorators.py @@ -0,0 +1,117 @@ +import logging +import traceback +from functools import partial, wraps, update_wrapper +from multiprocessing import Process +from threading import Thread + +from celery import shared_task as celery_shared_task +from celery import states +from celery.decorators import periodic_task as celery_periodic_task +from django.core.mail import mail_admins +from django.utils import timezone + +from orchestra.utils.db import close_connection +from orchestra.utils.python import AttrDict + +from .utils import get_name, get_id + + +logger = logging.getLogger(__name__) + + +def keep_state(fn): + """ logs task on djcelery's TaskState model """ + @wraps(fn) + def wrapper(*args, _task_id=None, _name=None, **kwargs): + from djcelery.models import TaskState + now = timezone.now() + if _task_id is None: + _task_id = get_id() + if _name is None: + _name = get_name(fn) + state = TaskState.objects.create( + state=states.STARTED, task_id=_task_id, name=_name, + args=str(args), kwargs=str(kwargs), tstamp=now) + try: + result = fn(*args, **kwargs) + except: + trace = traceback.format_exc() + subject = 'EXCEPTION executing task %s(args=%s, kwargs=%s)' % (_name, args, kwargs) + logger.error(subject) + logger.error(trace) + state.state = states.FAILURE + state.traceback = trace + state.runtime = (timezone.now()-now).total_seconds() + state.save() + mail_admins(subject, trace) + raise + else: + state.state = states.SUCCESS + state.result = str(result) + state.runtime = (timezone.now()-now).total_seconds() + state.save() + return result + return wrapper + + +def apply_async(fn, name=None, method='thread'): + """ replaces celery apply_async """ + def inner(fn, name, method, *args, **kwargs): + task_id = get_id() + kwargs.update({ + '_name': name, + '_task_id': task_id, + }) + thread = method(target=fn, args=args, kwargs=kwargs) + thread.start() + # Celery API compat + thread.request = AttrDict(id=task_id) + return thread + + if name is None: + name = get_name(fn) + if method == 'thread': + method = Thread + elif method == 'process': + method = Process + else: + raise NotImplementedError("%s concurrency method is not supported." % method) + fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method) + fn.delay = fn.apply_async + return fn + + +def task(fn=None, **kwargs): + # TODO override this if 'celerybeat' in sys.argv ? + from . import settings + # register task + if fn is None: + name = kwargs.get('name', None) + if settings.TASKS_BACKEND in ('thread', 'process'): + def decorator(fn): + return apply_async(celery_shared_task(**kwargs)(fn), name=name) + return decorator + else: + return celery_shared_task(**kwargs) + fn = celery_shared_task(fn) + if settings.TASKS_BACKEND in ('thread', 'process'): + fn = apply_async(fn) + return fn + + +def periodic_task(fn=None, **kwargs): + from . import settings + # register task + if fn is None: + name = kwargs.get('name', None) + if settings.TASKS_BACKEND in ('thread', 'process'): + def decorator(fn): + return apply_async(celery_periodic_task(**kwargs)(fn), name=name) + return decorator + else: + return celery_periodic_task(**kwargs) + fn = celery_periodic_task(fn) + if settings.TASKS_BACKEND in ('thread', 'process'): + name = kwargs.pop('name', None) + fn = update_wrapper(apply_async(fn, name), fn) + return fn diff --git a/orchestra/contrib/tasks/management/commands/beat.py b/orchestra/contrib/tasks/management/commands/beat.py new file mode 100644 index 00000000..ba73bc2c --- /dev/null +++ b/orchestra/contrib/tasks/management/commands/beat.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand + +from ... import beat + + +class Command(BaseCommand): + help = 'Runs periodic tasks.' + + def handle(self, *args, **options): + beat.run() diff --git a/orchestra/contrib/tasks/management/commands/runfunction.py b/orchestra/contrib/tasks/management/commands/runfunction.py new file mode 100644 index 00000000..a1b508ed --- /dev/null +++ b/orchestra/contrib/tasks/management/commands/runfunction.py @@ -0,0 +1,32 @@ +from django.core.management.base import BaseCommand + +from orchestra.utils.python import import_class + +from ... import keep_state, get_id, get_name + + +class Command(BaseCommand): + help = 'Runs Orchestra method.' + + def add_arguments(self, parser): + parser.add_argument('method', + help='Python path to the method to execute.') + parser.add_argument('args', nargs='*', + help='Additional arguments passed to the method.') + + def handle(self, *args, **options): + method = import_class(options['method']) + kwargs = {} + arguments = [] + for arg in args: + if '=' in args: + name, value = arg.split('=') + if value.isdigit(): + value = int(value) + kwargs[name] = value + else: + if arg.isdigit(): + arg = int(arg) + arguments.append(arg) + args = arguments + keep_state(method)(get_id(), get_name(method), *args, **kwargs) diff --git a/orchestra/contrib/tasks/management/commands/runtask.py b/orchestra/contrib/tasks/management/commands/runtask.py new file mode 100644 index 00000000..93aac361 --- /dev/null +++ b/orchestra/contrib/tasks/management/commands/runtask.py @@ -0,0 +1,48 @@ +import json + +from celery import current_app +from django.core.management.base import BaseCommand +from django.utils import timezone +from djcelery.models import PeriodicTask + +from ...decorators import keep_state + + +class Command(BaseCommand): + help = 'Runs Orchestra method.' + + def add_arguments(self, parser): + parser.add_argument('task', + help='Periodic task ID or task name.') + parser.add_argument('args', nargs='*', + help='Additional arguments passed to the task, when task name is used.') + + def handle(self, *args, **options): + task = options.get('task') + if task.isdigit(): + # periodic task + ptask = PeriodicTask.objects.get(pk=int(task)) + task = current_app.tasks[ptask.task] + args = json.loads(ptask.args) + kwargs = json.loads(ptask.kwargs) + ptask.last_run_at = timezone.now() + ptask.total_run_count += 1 + ptask.save() + else: + # task name + task = current_app.tasks[task] + kwargs = {} + arguments = [] + for arg in args: + if '=' in args: + name, value = arg.split('=') + if value.isdigit(): + value = int(value) + kwargs[name] = value + else: + if arg.isdigit(): + arg = int(arg) + arguments.append(arg) + args = arguments + # Run task synchronously, but logging TaskState + keep_state(task)(*args, **kwargs) diff --git a/orchestra/contrib/tasks/management/commands/syncperiodictasks.py b/orchestra/contrib/tasks/management/commands/syncperiodictasks.py new file mode 100644 index 00000000..7e9dfc8a --- /dev/null +++ b/orchestra/contrib/tasks/management/commands/syncperiodictasks.py @@ -0,0 +1,15 @@ +from django.core.management.base import BaseCommand +from djcelery.app import app +from djcelery.schedulers import DatabaseScheduler + + +class Command(BaseCommand): + help = 'Runs Orchestra method.' + + def handle(self, *args, **options): + dbschedule = DatabaseScheduler(app=app) + self.stdout.write('\033[1m%i periodic tasks have been syncronized:\033[0m' % len(dbschedule.schedule)) + size = max([len(name) for name in dbschedule.schedule])+1 + for name, task in dbschedule.schedule.items(): + spaces = ' '*(size-len(name)) + self.stdout.write(' %s%s%s' % (name, spaces, task.schedule)) diff --git a/orchestra/contrib/tasks/parser.py b/orchestra/contrib/tasks/parser.py new file mode 100644 index 00000000..23cc2fae --- /dev/null +++ b/orchestra/contrib/tasks/parser.py @@ -0,0 +1,61 @@ +import os + + +# Rename module to handler.py +class CronHandler(object): + def __init__(self, filename): + self.content = None + self.filename = filename + + def read(self): + comments = [] + self.content = [] + with open(self.filename, 'r') as handler: + for line in handler.readlines(): + line = line.strip() + if line.startswith('#'): + comments.append(line) + else: + schedule = line.split()[:5] + command = ' '.join(line.split()[5:]).strip() + self.content.append((schedule, command, comments)) + comments = [] + + def save(self, backup=True): + if self.content is None: + raise Exception("First read() the cron file!") + if backup: + os.rename(self.filename, self.filename + '.backup') + with open(self.filename, 'w') as handler: + handler.write('\n'.join(self.content)) + handler.truncate() + self.reload() + + def reload(self): + pass + # TODO + + def remove(self, command): + if self.content is None: + raise Exception("First read() the cron file!") + new_content = [] + for c_schedule, c_command, c_comments in self.content: + if command != c_command: + new_content.append((c_schedule, c_command, c_comments)) + self.content = new_content + + def add_or_update(self, schedule, command, comments=None): + """ if content contains an equal command, its schedule is updated """ + if self.content is None: + raise Exception("First read() the cron file!") + new_content = [] + replaced = False + for c_schedule, c_command, c_comments in self.content: + if command == c_command: + replaced = True + new_content.append((schedule, command, comments or c_comments)) + else: + new_content.append((c_schedule, c_command, c_comments)) + if not replaced: + new_content.append((schedule, command, comments or [])) + self.content = new_content diff --git a/orchestra/contrib/tasks/schedules.py b/orchestra/contrib/tasks/schedules.py new file mode 100644 index 00000000..c5af3bff --- /dev/null +++ b/orchestra/contrib/tasks/schedules.py @@ -0,0 +1,118 @@ +#import re + + +#class CronTab(object): +# pass + + +#class ParseException(Exception): +# """Raised by crontab_parser when the input can't be parsed.""" + + +## https://github.com/celery/celery/blob/master/celery/schedules.py +#class CrontabParser(object): +# """Parser for crontab expressions. Any expression of the form 'groups' +# (see BNF grammar below) is accepted and expanded to a set of numbers. +# These numbers represent the units of time that the crontab needs to +# run on:: +# digit :: '0'..'9' +# dow :: 'a'..'z' +# number :: digit+ | dow+ +# steps :: number +# range :: number ( '-' number ) ? +# numspec :: '*' | range +# expr :: numspec ( '/' steps ) ? +# groups :: expr ( ',' expr ) * +# The parser is a general purpose one, useful for parsing hours, minutes and +# day_of_week expressions. Example usage:: +# >>> minutes = crontab_parser(60).parse('*/15') +# [0, 15, 30, 45] +# >>> hours = crontab_parser(24).parse('*/4') +# [0, 4, 8, 12, 16, 20] +# >>> day_of_week = crontab_parser(7).parse('*') +# [0, 1, 2, 3, 4, 5, 6] +# It can also parse day_of_month and month_of_year expressions if initialized +# with an minimum of 1. Example usage:: +# >>> days_of_month = crontab_parser(31, 1).parse('*/3') +# [1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31] +# >>> months_of_year = crontab_parser(12, 1).parse('*/2') +# [1, 3, 5, 7, 9, 11] +# >>> months_of_year = crontab_parser(12, 1).parse('2-12/2') +# [2, 4, 6, 8, 10, 12] +# The maximum possible expanded value returned is found by the formula:: +# max_ + min_ - 1 +# """ +# ParseException = ParseException + +# _range = r'(\w+?)-(\w+)' +# _steps = r'/(\w+)?' +# _star = r'\*' + +# def __init__(self, max_=60, min_=0): +# self.max_ = max_ +# self.min_ = min_ +# self.pats = ( +# (re.compile(self._range + self._steps), self._range_steps), +# (re.compile(self._range), self._expand_range), +# (re.compile(self._star + self._steps), self._star_steps), +# (re.compile('^' + self._star + '$'), self._expand_star), +# ) + +# def parse(self, spec): +# acc = set() +# for part in spec.split(','): +# if not part: +# raise self.ParseException('empty part') +# acc |= set(self._parse_part(part)) +# return acc + +# def _parse_part(self, part): +# for regex, handler in self.pats: +# m = regex.match(part) +# if m: +# return handler(m.groups()) +# return self._expand_range((part, )) + +# def _expand_range(self, toks): +# fr = self._expand_number(toks[0]) +# if len(toks) > 1: +# to = self._expand_number(toks[1]) +# if to < fr: # Wrap around max_ if necessary +# return (list(range(fr, self.min_ + self.max_)) + +# list(range(self.min_, to + 1))) +# return list(range(fr, to + 1)) +# return [fr] + +# def _range_steps(self, toks): +# if len(toks) != 3 or not toks[2]: +# raise self.ParseException('empty filter') +# return self._expand_range(toks[:2])[::int(toks[2])] + +# def _star_steps(self, toks): +# if not toks or not toks[0]: +# raise self.ParseException('empty filter') +# return self._expand_star()[::int(toks[0])] + +# def _expand_star(self, *args): +# return list(range(self.min_, self.max_ + self.min_)) + +# def _expand_number(self, s): +# if isinstance(s, str) and s[0] == '-': +# raise self.ParseException('negative numbers not supported') +# try: +# i = int(s) +# except ValueError: +# try: +# i = weekday(s) +# except KeyError: +# raise ValueError('Invalid weekday literal {0!r}.'.format(s)) + +# max_val = self.min_ + self.max_ - 1 +# if i > max_val: +# raise ValueError( +# 'Invalid end range: {0} > {1}.'.format(i, max_val)) +# if i < self.min_: +# raise ValueError( +# 'Invalid beginning range: {0} < {1}.'.format(i, self.min_)) + +# return i diff --git a/orchestra/contrib/tasks/settings.py b/orchestra/contrib/tasks/settings.py new file mode 100644 index 00000000..03adc5f2 --- /dev/null +++ b/orchestra/contrib/tasks/settings.py @@ -0,0 +1,23 @@ +from orchestra.contrib.settings import Setting + + +TASKS_BACKEND = Setting('TASKS_BACKEND', + 'thread', + choices=( + ('thread', "threading.Thread (no queue)"), + ('process', "multiprocess.Process (no queue)"), + ('celery', "Celery (with queue)"), + ) +) + + +TASKS_ENABLE_UWSGI_CRON_BEAT = Setting('TASKS_ENABLE_UWSGI_CRON_BEAT', + False, + help_text="Not implemented.", +) + + + +TASKS_BACKEND_CLEANUP_DAYS = Setting('TASKS_BACKEND_CLEANUP_DAYS', + 10, +) diff --git a/orchestra/contrib/tasks/tasks.py b/orchestra/contrib/tasks/tasks.py new file mode 100644 index 00000000..2b377583 --- /dev/null +++ b/orchestra/contrib/tasks/tasks.py @@ -0,0 +1,14 @@ +from datetime import timedelta + +from celery.task.schedules import crontab +from django.utils import timezone +from djcelery.models import TaskState + +from . import periodic_task, settings + + +@periodic_task(run_every=crontab(hour=6, minute=0)) +def backend_logs_cleanup(): + days = settings.TASKS_BACKEND_CLEANUP_DAYS + epoch = timezone.now()-timedelta(days=days) + return TaskState.objects.filter(tstamp__lt=epoch).only('id').delete() diff --git a/orchestra/contrib/tasks/utils.py b/orchestra/contrib/tasks/utils.py new file mode 100644 index 00000000..96b5bf0b --- /dev/null +++ b/orchestra/contrib/tasks/utils.py @@ -0,0 +1,19 @@ +import threading +from uuid import uuid4 + +from orchestra.utils.db import close_connection + + +def get_id(): + return str(uuid4()) + + +def get_name(fn): + return '.'.join((fn.__module__, fn.__name__)) + + +def run(method, *args, **kwargs): + run_async = kwargs.pop('run_async', True) + thread = threading.Thread(target=close_connection(method), args=args, kwargs=kwargs) + thread = Process(target=close_connection(counter)) + thread.start() diff --git a/orchestra/contrib/vps/__init__.py b/orchestra/contrib/vps/__init__.py new file mode 100644 index 00000000..96cf9729 --- /dev/null +++ b/orchestra/contrib/vps/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.vps.apps.VPSConfig' diff --git a/orchestra/contrib/vps/admin.py b/orchestra/contrib/vps/admin.py new file mode 100644 index 00000000..346ddaf9 --- /dev/null +++ b/orchestra/contrib/vps/admin.py @@ -0,0 +1,47 @@ +from django.contrib import admin +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter +from orchestra.forms import UserCreationForm, NonStoredUserChangeForm + +from .models import VPS + + +class VPSAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdmin): + list_display = ('hostname', 'type', 'template', 'display_active', 'account_link') + list_filter = ('type', IsActiveListFilter, 'template') + form = NonStoredUserChangeForm + add_form = UserCreationForm + readonly_fields = ('account_link',) + search_fields = ('hostname', 'account__username', 'template') + change_readonly_fields = ('account', 'hostname', 'type', 'template') + fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('account_link', 'hostname', 'type', 'template', 'is_active') + }), + (_("Login"), { + 'classes': ('wide',), + 'fields': ('password',) + }) + ) + add_fieldsets = ( + (None, { + 'classes': ('wide',), + 'fields': ('account', 'hostname', 'type', 'template') + }), + (_("Login"), { + 'classes': ('wide',), + 'fields': ('password1', 'password2',) + }), + ) + actions = (list_accounts,) + + def get_change_password_username(self, obj): + return 'root@%s' % obj.hostname + + +admin.site.register(VPS, VPSAdmin) diff --git a/orchestra/contrib/vps/apps.py b/orchestra/contrib/vps/apps.py new file mode 100644 index 00000000..919bb54d --- /dev/null +++ b/orchestra/contrib/vps/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class VPSConfig(AppConfig): + name = 'orchestra.contrib.vps' + verbose_name = 'VPS' + + def ready(self): + from .models import VPS + services.register(VPS, icon='TuxBox.png') diff --git a/orchestra/contrib/vps/backends.py b/orchestra/contrib/vps/backends.py new file mode 100644 index 00000000..8b817b40 --- /dev/null +++ b/orchestra/contrib/vps/backends.py @@ -0,0 +1,154 @@ +import decimal +import textwrap + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor + +from . import settings + + +class ProxmoxOVZ(ServiceController): + model = 'vps.VPS' + + RESOURCES = ( + ('memory', 'mem'), + ('swap', 'swap'), + ('disk', 'disk') + ) + GET_PROXMOX_INFO = textwrap.dedent(""" + function get_vz_info () { + hostname=$1 + version=$(pveversion | cut -d '/' -f2 | cut -d'.' -f1) + if [[ $version -lt 2 ]]; then + conf=$(grep "CID\\|:$hostname:" /var/lib/pve-manager/vzlist | grep -B1 ":$hostname:") + CID=$(echo "$conf" | head -n1 | cut -d':' -f2) + CTID=$(echo "$conf" | tail -n1 | cut -d':' -f1) + node=$(pveca -l | grep "^\\s*$CID\\s*:" | awk {'print $3'}) + else + conf=$(grep -r "HOSTNAME=\\"$hostname\\"" /etc/pve/nodes/*/openvz/*.conf) + node=$(echo "${conf}" | cut -d"/" -f5) + CTID=$(echo "${conf}" | cut -d"/" -f7 | cut -d"\\." -f1) + fi + echo $CTID $node + }""") + + def prepare(self): + super(ProxmoxOVZ, self).prepare() + self.append(self.GET_PROXMOX_INFO) + + def get_vzset_args(self, context): + args = list(settings.VPS_DEFAULT_VZSET_ARGS) + for resource, arg_name in self.RESOURCES: + try: + allocation = context[resource] + except KeyError: + pass + else: + args.append('--%s %i' % (arg_name, allocation)) + return ' '.join(args) + + def run_ssh_commands(self, ssh_commands): + commands = '\n '.join(ssh_commands) + self.append(textwrap.dedent("""\ + cat << EOF | ssh root@${info[1]} + %s + EOF""") % commands + ) + + def save(self, vps): + # TODO create the container + context = self.get_context(vps) + self.append(textwrap.dedent(""" + info=( $(get_vz_info %(hostname)s) ) + echo "Managing ${info[@]}"\ + """) % context + ) + ssh_commands = [] + vzset_args = self.get_vzset_args(context) + if vzset_args: + context['vzset_args'] = vzset_args + ssh_commands.append("pvectl vzset ${info[0]} %(vzset_args)s" % context) + if hasattr(vps, 'password'): + context['password'] = vps.password.replace('$', '\\$') + ssh_commands.append(textwrap.dedent("""\ + echo 'root:%(password)s' \\ + | chroot /var/lib/vz/private/${info[0]} chpasswd -e""") % context + ) + self.run_ssh_commands(ssh_commands) + + def get_context(self, vps): + context = { + 'hostname': vps.hostname, + } + for resource, __ in self.RESOURCES: + try: + allocation = getattr(vps.resources, resource).allocated + except AttributeError: + pass + else: + context[resource] = allocation + return context + + +class ProxmoxOpenVZTraffic(ServiceMonitor): + model = 'vps.VPS' + resource = ServiceMonitor.TRAFFIC + monthly_sum_old_values = True + GET_PROXMOX_INFO = ProxmoxOVZ.GET_PROXMOX_INFO + + def prepare(self): + super(ProxmoxOpenVZTraffic, self).prepare() + self.append(self.GET_PROXMOX_INFO) + self.append(textwrap.dedent(""" + function monitor () { + object_id=$1 + hostname=$2 + info=( $(get_vz_info $hostname) ) + cat << EOF | ssh root@${info[1]} + vzctl exec ${info[0]} cat /proc/net/dev \\ + | grep venet0 \\ + | tr ':' ' ' \\ + | awk '{sum=\\$2+\\$10} END {printf ("$object_id %0.0f\\n", sum)}' + EOF + } + """) + ) + + def process(self, line): + """ diff with last stored state """ + object_id, value, state = super(ProxmoxOpenVZTraffic, self).process(line) + value = decimal.Decimal(value) + last = self.get_last_data(object_id) + if not last or last.state > value: + return object_id, value, value + return object_id, value-last.state, value + + def monitor(self, vps): + """ Get OpenVZ container traffic on a Proxmox cluster """ + context = self.get_context(vps) + self.append('monitor %(object_id)s %(hostname)s' % context) + + def get_context(self, vps): + return { + 'object_id': vps.id, + 'hostname': vps.hostname, + } + + +class LxcController(ServiceController): + model = 'vps.VPS' + + RESOURCES = ( + ('memory', 'mem'), + ('disk', 'disk'), + ('vcpu', 'vcpu') + ) + + def prepare(self): + super(LxcController, self).prepare() + + def save(self, vps): + # TODO create the container + pass + + diff --git a/orchestra/contrib/vps/backends.py.new b/orchestra/contrib/vps/backends.py.new new file mode 100644 index 00000000..2ead22c3 --- /dev/null +++ b/orchestra/contrib/vps/backends.py.new @@ -0,0 +1,135 @@ +import decimal +import textwrap + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor + +from . import settings + + +class ProxmoxOVZ(ServiceController): + model = 'vps.VPS' + + RESOURCES = ( + ('memory', 'mem'), + ('swap', 'swap'), + ('disk', 'disk') + ) + GET_PROXMOX_INFO = textwrap.dedent(""" + function get_vz_info () { + hostname=$1 + version=$(pveversion | cut -d '/' -f2 | cut -d'.' -f1) + if [[ $version -lt 2 ]]; then + conf=$(grep "CID\\|:$hostname:" /var/lib/pve-manager/vzlist | grep -B1 ":$hostname:") + CID=$(echo "$conf" | head -n1 | cut -d':' -f2) + CTID=$(echo "$conf" | tail -n1 | cut -d':' -f1) + node=$(pveca -l | grep "^\\s*$CID\\s*:" | awk {'print $3'}) + else + conf=$(grep -r "HOSTNAME=\\"$hostname\\"" /etc/pve/nodes/*/openvz/*.conf) + node=$(echo "${conf}" | cut -d"/" -f5) + CTID=$(echo "${conf}" | cut -d"/" -f7 | cut -d"\\." -f1) + fi + echo $CTID $node + }""") + + def prepare(self): + super(ProxmoxOVZ, self).prepare() + self.append(self.GET_PROXMOX_INFO) + + def get_vzset_args(self, context): + args = list(settings.VPS_DEFAULT_VZSET_ARGS) + for resource, arg_name in self.RESOURCES: + try: + allocation = context[resource] + except KeyError: + pass + else: + args.append('--%s %i' % (arg_name, allocation)) + return ' '.join(args) + + def run_ssh_commands(self, ssh_commands): + commands = '\n '.join(ssh_commands) + self.append(textwrap.dedent("""\ + cat << EOF | ssh root@${info[1]} + %s + EOF""") % commands + ) + + def save(self, vps): + # TODO create the container + context = self.get_context(vps) + self.append(textwrap.dedent(""" + info=( $(get_vz_info %(hostname)s) ) + echo "Managing ${info[@]}"\ + """) % context + ) + ssh_commands = [] + vzset_args = self.get_vzset_args(context) + if vzset_args: + context['vzset_args'] = vzset_args + ssh_commands.append("pvectl vzset ${info[0]} %(vzset_args)s" % context) + if hasattr(vps, 'password'): + context['password'] = vps.password.replace('$', '\\$') + ssh_commands.append(textwrap.dedent("""\ + echo 'root:%(password)s' \\ + | chroot /var/lib/vz/private/${info[0]} chpasswd -e""") % context + ) + self.run_ssh_commands(ssh_commands) + + def get_context(self, vps): + context = { + 'hostname': vps.hostname, + } + for resource, __ in self.RESOURCES: + try: + allocation = getattr(vps.resources, resource).allocated + except AttributeError: + pass + else: + context[resource] = allocation + return context + + +class ProxmoxOpenVZTraffic(ServiceMonitor): + model = 'vps.VPS' + resource = ServiceMonitor.TRAFFIC + monthly_sum_old_values = True + GET_PROXMOX_INFO = ProxmoxOVZ.GET_PROXMOX_INFO + + def prepare(self): + super(ProxmoxOpenVZTraffic, self).prepare() + self.append(self.GET_PROXMOX_INFO) + self.append(textwrap.dedent(""" + function monitor () { + object_id=$1 + hostname=$2 + info=( $(get_vz_info $hostname) ) + cat << EOF | ssh root@${info[1]} + vzctl exec ${info[0]} cat /proc/net/dev \\ + | grep venet0 \\ + | tr ':' ' ' \\ + | awk '{sum=\\$2+\\$10} END {printf ("$object_id %0.0f\\n", sum)}' + EOF + } + """) + ) + + def process(self, line): + """ diff with last stored state """ + object_id, value, state = super(ProxmoxOpenVZTraffic, self).process(line) + value = decimal.Decimal(value) + last = self.get_last_data(object_id) + if not last or last.state > value: + return object_id, value, value + return object_id, value-last.state, value + + def monitor(self, vps): + """ Get OpenVZ container traffic on a Proxmox cluster """ + context = self.get_context(vps) + self.append('monitor %(object_id)s %(hostname)s' % context) + + def get_context(self, vps): + return { + 'object_id': vps.id, + 'hostname': vps.hostname, + } diff --git a/orchestra/contrib/vps/models.py b/orchestra/contrib/vps/models.py new file mode 100644 index 00000000..f5b19f7b --- /dev/null +++ b/orchestra/contrib/vps/models.py @@ -0,0 +1,46 @@ +from django.contrib.auth.hashers import make_password +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from orchestra.core.validators import validate_hostname + +from . import settings + + +class VPS(models.Model): + hostname = models.CharField(_("hostname"), max_length=256, unique=True, + validators=[validate_hostname]) + type = models.CharField(_("type"), max_length=64, choices=settings.VPS_TYPES, + default=settings.VPS_DEFAULT_TYPE) + template = models.CharField(_("template"), max_length=64, + choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE, + help_text=_("Initial template.")) + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='vpss') + is_active = models.BooleanField(_("active"), default=True) + + class Meta: + verbose_name = "VPS" + verbose_name_plural = "VPSs" + + def __str__(self): + return self.hostname + + def set_password(self, raw_password): + self.password = make_password(raw_password) + + def get_username(self): + return self.hostname + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + @property + def active(self): + return self.is_active and self.account.is_active + diff --git a/orchestra/contrib/vps/settings.py b/orchestra/contrib/vps/settings.py new file mode 100644 index 00000000..ec0e2a1a --- /dev/null +++ b/orchestra/contrib/vps/settings.py @@ -0,0 +1,36 @@ +from orchestra.contrib.settings import Setting + + +VPS_TYPES = Setting('VPS_TYPES', + ( + ('openvz', 'OpenVZ container'), + ('lxc', 'LXC container') + ), + validators=[Setting.validate_choices] +) + + +VPS_DEFAULT_TYPE = Setting('VPS_DEFAULT_TYPE', + 'lxc', + choices=VPS_TYPES +) + + +VPS_TEMPLATES = Setting('VPS_TEMPLATES', + ( + ('debian7', 'Debian 7 - Wheezy'), + ('placeholder', 'LXC placeholder') + ), + validators=[Setting.validate_choices] +) + + +VPS_DEFAULT_TEMPLATE = Setting('VPS_DEFAULT_TEMPLATE', + 'placeholder', + choices=VPS_TEMPLATES +) + + +VPS_DEFAULT_VZSET_ARGS = Setting('VPS_DEFAULT_VZSET_ARGS', + ('--onboot yes',), +) diff --git a/orchestra/contrib/webapps/__init__.py b/orchestra/contrib/webapps/__init__.py new file mode 100644 index 00000000..f08acd2d --- /dev/null +++ b/orchestra/contrib/webapps/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.webapps.apps.WebAppsConfig' diff --git a/orchestra/contrib/webapps/admin.py b/orchestra/contrib/webapps/admin.py new file mode 100644 index 00000000..d1dc7a18 --- /dev/null +++ b/orchestra/contrib/webapps/admin.py @@ -0,0 +1,97 @@ +from django import forms +from django.contrib import admin +from django.urls import reverse +from django.utils.encoding import force_str +from django.utils.safestring import mark_safe +from django.utils.translation import gettext, gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.utils import admin_link, get_modeladmin +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.forms.widgets import DynamicHelpTextSelect +from orchestra.plugins.admin import SelectPluginAdminMixin, display_plugin_field +from orchestra.utils.html import get_on_site_link + +from .filters import HasWebsiteListFilter, DetailListFilter +from .models import WebApp, WebAppOption +from .options import AppOption +from .types import AppType + + +class WebAppOptionInline(admin.TabularInline): + model = WebAppOption + extra = 1 + + OPTIONS_HELP_TEXT = { + op.name: force_str(op.help_text) for op in AppOption.get_plugins() + } + + class Media: + css = { + 'all': ('orchestra/css/hide-inline-id.css',) + } + + def formfield_for_dbfield(self, db_field, **kwargs): + if db_field.name == 'value': + kwargs['widget'] = forms.TextInput(attrs={'size':'100'}) + if db_field.name == 'name': + if self.parent_object: + plugin = self.parent_object.type_class + else: + request = kwargs['request'] + webapp_modeladmin = get_modeladmin(self.parent_model) + plugin_value = webapp_modeladmin.get_plugin_value(request) + plugin = AppType.get(plugin_value) + kwargs['choices'] = plugin.get_group_options_choices() + # Help text based on select widget + target = 'this.id.replace("name", "value")' + kwargs['widget'] = DynamicHelpTextSelect(target, self.OPTIONS_HELP_TEXT) + return super(WebAppOptionInline, self).formfield_for_dbfield(db_field, **kwargs) + + +class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'name', 'display_type', 'display_detail', 'display_websites', 'account_link' + ) + list_filter = ('type', HasWebsiteListFilter, DetailListFilter) + inlines = [WebAppOptionInline] + readonly_fields = ('account_link',) + change_readonly_fields = ('name', 'type', 'display_websites') + search_fields = ('name', 'account__username', 'data', 'website__domains__name') + list_prefetch_related = ('content_set__website', 'content_set__website__domains') + plugin = AppType + plugin_field = 'type' + plugin_title = _("Web application type") + actions = (list_accounts,) + + display_type = display_plugin_field('type') + + @mark_safe + def display_websites(self, webapp): + websites = [] + for content in webapp.content_set.all(): + site_url = content.get_absolute_url() + site_link = get_on_site_link(site_url) + website = content.website + #name = "%s on %s %s" % (website.name, content.path, site_link) + name = "%s on %s" % (website.name, content.path) + link = admin_link(display=name)(website) + websites.append(link) + if not websites: + add_url = reverse('admin:websites_website_add') + add_url += '?account=%s' % webapp.account_id + plus = '+' + websites.append('%s%s' % (add_url, plus, gettext("Add website"))) + return '
    '.join(websites) + display_websites.short_description = _("web sites") + + def display_detail(self, webapp): + try: + return webapp.type_instance.get_detail() + except KeyError: + return mark_safe("Not available") + display_detail.short_description = _("detail") + + +admin.site.register(WebApp, WebAppAdmin) diff --git a/orchestra/contrib/webapps/api.py b/orchestra/contrib/webapps/api.py new file mode 100644 index 00000000..9ba43050 --- /dev/null +++ b/orchestra/contrib/webapps/api.py @@ -0,0 +1,50 @@ +from rest_framework import viewsets + +from orchestra.api import router, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from . import settings +from .models import WebApp +from .options import AppOption +from .serializers import WebAppSerializer +from .types import AppType + + +class WebAppViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = WebApp.objects.prefetch_related('options').all() + serializer_class = WebAppSerializer + filter_fields = ('name',) + + def options(self, request): + metadata = super(WebAppViewSet, self).options(request) + names = [ + 'WEBAPPS_BASE_DIR', 'WEBAPPS_TYPES', 'WEBAPPS_WEBAPP_OPTIONS', + 'WEBAPPS_PHP_DISABLED_FUNCTIONS', 'WEBAPPS_DEFAULT_TYPE' + ] + metadata.data['settings'] = { + name.lower(): getattr(settings, name, None) for name in names + } + # AppTypes + meta = self.metadata_class() + app_types = {} + for app_type in AppType.get_plugins(): + if app_type.serializer: + data = meta.get_serializer_info(app_type.serializer()) + else: + data = {} + data['option_groups'] = app_type.option_groups + app_types[app_type.get_name()] = data + metadata.data['actions']['types'] = app_types + # Options + options = {} + for option in AppOption.get_plugins(): + options[option.get_name()] = { + 'verbose_name': option.get_verbose_name(), + 'help_text': option.help_text, + 'group': option.group, + } + metadata.data['actions']['options'] = options + return metadata + + +router.register(r'webapps', WebAppViewSet) diff --git a/orchestra/contrib/webapps/apps.py b/orchestra/contrib/webapps/apps.py new file mode 100644 index 00000000..b183c2c8 --- /dev/null +++ b/orchestra/contrib/webapps/apps.py @@ -0,0 +1,13 @@ +from django.apps import AppConfig + +from orchestra.core import services + + +class WebAppsConfig(AppConfig): + name = 'orchestra.contrib.webapps' + verbose_name = 'Webapps' + + def ready(self): + from .models import WebApp + services.register(WebApp, icon='Applications-other.png') + from . import signals diff --git a/orchestra/contrib/webapps/backends/__init__.py b/orchestra/contrib/webapps/backends/__init__.py new file mode 100644 index 00000000..57c2c77e --- /dev/null +++ b/orchestra/contrib/webapps/backends/__init__.py @@ -0,0 +1,75 @@ +import pkgutil +import textwrap + +from .. import settings + + +class WebAppServiceMixin(object): + model = 'webapps.WebApp' + related_models = ( + ('webapps.WebAppOption', 'webapp'), + ) + directive = None + doc_settings = (settings, + ('WEBAPPS_UNDER_CONSTRUCTION_PATH', 'WEBAPPS_MOVE_ON_DELETE_PATH',) + ) + + def create_webapp_dir(self, context): + self.append(textwrap.dedent(""" + # Create webapp dir + CREATED=0 + if [[ ! -e %(app_path)s ]]; then + CREATED=1 + mkdir -p %(app_path)s + chown %(user)s:%(group)s %(app_path)s + fi""") % context + ) + + def set_under_construction(self, context): + if context['under_construction_path']: + self.append(textwrap.dedent(""" + # Set under construction if needed + if [[ $CREATED == 1 && ! $(ls -A %(app_path)s | head -n1) ]]; then + # Async wait some seconds for other backends to lock app_path or cp under construction + nohup bash -c ' + sleep 2 + if [[ ! $(ls -A %(app_path)s | head -n1) ]]; then + cp -r %(under_construction_path)s %(app_path)s + chown -R %(user)s:%(group)s %(app_path)s/* + fi' &> /dev/null & + fi""") % context + ) + + def delete_webapp_dir(self, context): + if context['deleted_app_path']: + self.append(textwrap.dedent("""\ + # Move app into WEBAPPS_MOVE_ON_DELETE_PATH, nesting if exists. + deleted_app_path="%(deleted_app_path)s" + while [[ -e $deleted_app_path ]]; do + deleted_app_path="${deleted_app_path}/$(basename ${deleted_app_path})" + done + mv %(app_path)s $deleted_app_path || exit_code=$? + """) % context + ) + else: + self.append("rm -fr %(app_path)s" % context) + + def get_context(self, webapp): + context = webapp.type_instance.get_directive_context() + context.update({ + 'user': webapp.get_username(), + 'group': webapp.get_groupname(), + 'app_name': webapp.name, + 'app_type': webapp.type, + 'app_path': webapp.get_path(), + 'banner': self.get_banner(), + 'under_construction_path': settings.WEBAPPS_UNDER_CONSTRUCTION_PATH, + 'is_mounted': webapp.content_set.exists(), + }) + context['deleted_app_path'] = settings.WEBAPPS_MOVE_ON_DELETE_PATH % context + return context + + +for __, module_name, __ in pkgutil.walk_packages(__path__): + # sorry for the exec(), but Import module function fails :( + exec('from . import %s' % module_name) diff --git a/orchestra/contrib/webapps/backends/moodle.py b/orchestra/contrib/webapps/backends/moodle.py new file mode 100644 index 00000000..5579e8a4 --- /dev/null +++ b/orchestra/contrib/webapps/backends/moodle.py @@ -0,0 +1,108 @@ +import os +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace + +from .. import settings + +from . import WebAppServiceMixin + + +class MoodleController(WebAppServiceMixin, ServiceController): + """ + Installs the latest version of Moodle available on download.moodle.org + """ + verbose_name = _("Moodle") + model = 'webapps.WebApp' + default_route_match = "webapp.type == 'moodle-php'" + doc_settings = (settings, + ('WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST',) + ) + + + def save(self, webapp): + context = self.get_context(webapp) + self.append(textwrap.dedent("""\ + if [[ $(ls "%(app_path)s" | wc -l) -gt 1 ]]; then + echo "App directory not empty." 2> /dev/null + exit 0 + fi + mkdir -p %(app_path)s + # Prevent other backends from writting here + touch %(app_path)s/.lock + # Weekly caching + moodle_date=$(date -r $(readlink %(cms_cache_dir)s/moodle) +%%s || echo 0) + if [[ $moodle_date -lt $(($(date +%%s)-7*24*60*60)) ]]; then + moodle_url=$(wget https://download.moodle.org/releases/latest/ -O - -q \\ + | tr ' ' '\\n' \\ + | grep 'moodle-latest.*.tgz"' \\ + | sed -E 's#href="([^"]+)".*#\\1#' \\ + | head -n 1 \\ + | sed "s#download.php/#download.php/direct/#") + filename=${moodle_url##*/} + wget $moodle_url -O - --no-check-certificate \\ + | tee %(cms_cache_dir)s/$filename \\ + | tar -xzvf - -C %(app_path)s --strip-components=1 + rm -f %(cms_cache_dir)s/moodle + ln -s %(cms_cache_dir)s/$filename %(cms_cache_dir)s/moodle + else + tar -xzvf %(cms_cache_dir)s/moodle -C %(app_path)s --strip-components=1 + fi + mkdir %(app_path)s/moodledata && { + chmod 750 %(app_path)s/moodledata + echo -n 'order deny,allow\\ndeny from all' > %(app_path)s/moodledata/.htaccess + } + if [[ ! -e %(app_path)s/config.php ]]; then + cp %(app_path)s/config-dist.php %(app_path)s/config.php + sed -i "s#dbtype\s*= '.*#dbtype = '%(db_type)s';#" %(app_path)s/config.php + sed -i "s#dbhost\s*= '.*#dbhost = '%(db_host)s';#" %(app_path)s/config.php + sed -i "s#dbname\s*= '.*#dbname = '%(db_name)s';#" %(app_path)s/config.php + sed -i "s#dbuser\s*= '.*#dbuser = '%(db_user)s';#" %(app_path)s/config.php + sed -i "s#dbpass\s*= '.*#dbpass = '%(password)s';#" %(app_path)s/config.php + sed -i "s#dataroot\s*= '.*#dataroot = '%(app_path)s/moodledata';#" %(app_path)s/config.php + sed -i "s#wwwroot\s*= '.*#wwwroot = '%(www_root)s';#" %(app_path)s/config.php + + fi + rm %(app_path)s/.lock + chown -R %(user)s:%(group)s %(app_path)s + # Run install moodle cli command on the background, because it takes so long... + stdout=$(mktemp) + stderr=$(mktemp) + nohup su - %(user)s --shell /bin/bash << 'EOF' > $stdout 2> $stderr & + php %(app_path)s/admin/cli/install_database.php \\ + --fullname="%(site_name)s" \\ + --shortname="%(site_name)s" \\ + --adminpass="%(password)s" \\ + --adminemail="%(email)s" \\ + --non-interactive \\ + --agree-license \\ + --allow-unstable + EOF + pid=$! + sleep 2 + if ! ps -p $pid > /dev/null; then + cat $stdout + cat $stderr >&2 + exit_code=$(wait $pid) + fi + rm $stdout $stderr + """) % context + ) + + def get_context(self, webapp): + context = super(MoodleController, self).get_context(webapp) + contents = webapp.content_set.all() + context.update({ + 'db_type': 'mysqli', + 'db_name': webapp.data['db_name'], + 'db_user': webapp.data['db_user'], + 'password': webapp.data['password'], + 'db_host': settings.WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST, + 'email': webapp.account.email, + 'site_name': "%s Courses" % webapp.account.get_full_name(), + 'cms_cache_dir': os.path.normpath(settings.WEBAPPS_CMS_CACHE_DIR), + 'www_root': contents[0].website.get_absolute_url() if contents else 'http://empty' + }) + return replace(context, '"', "'") diff --git a/orchestra/contrib/webapps/backends/php.py b/orchestra/contrib/webapps/backends/php.py new file mode 100644 index 00000000..7478fc41 --- /dev/null +++ b/orchestra/contrib/webapps/backends/php.py @@ -0,0 +1,319 @@ +import os +import textwrap +from collections import OrderedDict + +from django.template import Template, Context +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController + +from . import WebAppServiceMixin +from .. import settings, utils + + +class PHPController(WebAppServiceMixin, ServiceController): + """ + PHP support for apache-mod-fcgid and php-fpm. + It handles switching between these two PHP process management systemes. + """ + MERGE = settings.WEBAPPS_MERGE_PHP_WEBAPPS + + verbose_name = _("PHP FPM/FCGID") + default_route_match = "webapp.type.endswith('php')" + doc_settings = (settings, ( + 'WEBAPPS_MERGE_PHP_WEBAPPS', + 'WEBAPPS_FPM_DEFAULT_MAX_CHILDREN', + 'WEBAPPS_PHP_CGI_BINARY_PATH', + 'WEBAPPS_PHP_CGI_RC_DIR', + 'WEBAPPS_PHP_CGI_INI_SCAN_DIR', + 'WEBAPPS_FCGID_CMD_OPTIONS_PATH', + 'WEBAPPS_PHPFPM_POOL_PATH', + 'WEBAPPS_PHP_MAX_REQUESTS', + )) + + def save(self, webapp): + self.delete_old_config(webapp) + context = self.get_context(webapp) + self.create_webapp_dir(context) + if webapp.type_instance.is_fpm: + self.save_fpm(webapp, context) + elif webapp.type_instance.is_fcgid: + self.save_fcgid(webapp, context) + else: + raise TypeError("Unknown PHP execution type") +# LEGACY CLEANUP FUNCTIONS. TODO REMOVE WHEN SURE NOT NEEDED. +# self.delete_fcgid(webapp, context, preserve=True) +# self.delete_fpm(webapp, context, preserve=True) + self.set_under_construction(context) + + def delete_config(self,webapp): + context = self.get_context(webapp) + to_delete = [] + if webapp.type_instance.is_fpm: + to_delete.append(settings.WEBAPPS_PHPFPM_POOL_PATH % context) + to_delete.append(settings.WEBAPPS_FPM_LISTEN % context) + elif webapp.type_instance.is_fcgid: + to_delete.append(settings.WEBAPPS_FCGID_WRAPPER_PATH % context) + to_delete.append(settings.WEBAPPS_FCGID_CMD_OPTIONS_PATH % context) + for item in to_delete: + self.append('rm -f "{}"'.format(item)) + + def delete_old_config(self,webapp): + # Check if we loaded the old version of the webapp. If so, we're updating + # rather than creating, so we must make sure the old config files are removed. + if hasattr(webapp, '_old_self'): + self.append("# Clean old configuration files") + self.delete_config(webapp._old_self) + else: + self.append("# No old config files to delete") + + def save_fpm(self, webapp, context): + self.append(textwrap.dedent(""" + # Generate FPM configuration + read -r -d '' fpm_config << 'EOF' || true + %(fpm_config)s + EOF + { + echo -e "${fpm_config}" | diff -N -I'^\s*;;' %(fpm_path)s - + } || { + echo -e "${fpm_config}" > %(fpm_path)s + UPDATED_FPM=1 + } + """) % context + ) + + def save_fcgid(self, webapp, context): + self.append("mkdir -p %(wrapper_dir)s" % context) + self.append(textwrap.dedent(""" + # Generate FCGID configuration + read -r -d '' wrapper << 'EOF' || true + %(wrapper)s + EOF + { + echo -e "${wrapper}" | diff -N -I'^\s*#' %(wrapper_path)s - + } || { + echo -e "${wrapper}" > %(wrapper_path)s + if [[ %(is_mounted)i -eq 1 ]]; then + # Reload fcgid wrapper (All PHP versions, because of version changing support) + pkill -SIGHUP -U %(user)s "^php[0-9\.]+-cgi$" || true + fi + } + chmod 550 %(wrapper_dir)s + chmod 550 %(wrapper_path)s + chown -R %(user)s:%(group)s %(wrapper_dir)s""") % context + ) + if context['cmd_options']: + self.append(textwrap.dedent("""\ + # FCGID options + read -r -d '' cmd_options << 'EOF' || true + %(cmd_options)s + EOF + { + echo -e "${cmd_options}" | diff -N -I'^\s*#' %(cmd_options_path)s - + } || { + echo -e "${cmd_options}" > %(cmd_options_path)s + [[ ${UPDATED_APACHE} -eq 0 ]] && UPDATED_APACHE=%(is_mounted)i + } + """ ) % context + ) + else: + self.append("rm -f %(cmd_options_path)s\n" % context) + + def delete(self, webapp): + context = self.get_context(webapp) + self.delete_old_config(webapp) +# if webapp.type_instance.is_fpm: +# self.delete_fpm(webapp, context) +# elif webapp.type_instance.is_fcgid: +# self.delete_fcgid(webapp, context) + self.delete_webapp_dir(context) + + def has_sibilings(self, webapp, context): + return type(webapp).objects.filter( + account=webapp.account_id, + data__contains='"php_version":"%s"' % context['php_version'], + ).exclude(id=webapp.pk).exists() + + def all_versions_to_delete(self, webapp, context, preserve=False): + context_copy = dict(context) + for php_version, verbose in settings.WEBAPPS_PHP_VERSIONS: + if preserve and php_version == context['php_version']: + continue + php_version_number = utils.extract_version_number(php_version) + context_copy['php_version'] = php_version + context_copy['php_version_number'] = php_version_number + if not self.MERGE or not self.has_sibilings(webapp, context_copy): + yield context_copy + + def delete_fpm(self, webapp, context, preserve=False): + """ delete all pools in order to efectively support changing php-fpm version """ + for context_copy in self.all_versions_to_delete(webapp, context, preserve): + context_copy['fpm_path'] = settings.WEBAPPS_PHPFPM_POOL_PATH % context_copy + self.append("rm -f %(fpm_path)s" % context_copy) + + def delete_fcgid(self, webapp, context, preserve=False): + """ delete all pools in order to efectively support changing php-fcgid version """ + for context_copy in self.all_versions_to_delete(webapp, context, preserve): + context_copy.update({ + 'wrapper_path': settings.WEBAPPS_FCGID_WRAPPER_PATH % context_copy, + 'cmd_options_path': settings.WEBAPPS_FCGID_CMD_OPTIONS_PATH % context_copy, + }) + self.append("rm -f %(wrapper_path)s" % context_copy) + self.append("rm -f %(cmd_options_path)s" % context_copy) + + def prepare(self): + super(PHPController, self).prepare() + self.append(textwrap.dedent(""" + BACKEND="PHPController" + echo "$BACKEND" >> /dev/shm/reload.apache2 + + function coordinate_apache_reload () { + # Coordinate Apache reload with other concurrent backends (e.g. Apache2Controller) + is_last=0 + counter=0 + while ! mv /dev/shm/reload.apache2 /dev/shm/reload.apache2.locked; do + sleep 0.1; + if [[ $counter -gt 4 ]]; then + echo "[ERROR]: Apache reload synchronization deadlocked!" >&2 + exit 10 + fi + counter=$(($counter+1)) + done + state="$(grep -v -E "^$BACKEND($|\s)" /dev/shm/reload.apache2.locked)" || is_last=1 + [[ $is_last -eq 0 ]] && { + echo "$state" | grep -v ' RELOAD$' || is_last=1 + } + if [[ $is_last -eq 1 ]]; then + echo "[DEBUG]: Last backend to run, update: $UPDATED_APACHE, state: '$state'" + if [[ $UPDATED_APACHE -eq 1 || "$state" =~ .*RELOAD$ ]]; then + if service apache2 status > /dev/null; then + service apache2 reload + else + service apache2 start + fi + fi + rm /dev/shm/reload.apache2.locked + else + echo "$state" > /dev/shm/reload.apache2.locked + if [[ $UPDATED_APACHE -eq 1 ]]; then + echo -e "[DEBUG]: Apache will be reloaded by another backend:\\n${state}" + echo "$BACKEND RELOAD" >> /dev/shm/reload.apache2.locked + fi + mv /dev/shm/reload.apache2.locked /dev/shm/reload.apache2 + fi + }""") + ) + + def commit(self): + context = { + 'reload_pool': settings.WEBAPPS_PHPFPM_RELOAD_POOL, + } + self.append(textwrap.dedent(""" + # Apply changes if needed + if [[ $UPDATED_FPM -eq 1 ]]; then + %(reload_pool)s + fi + coordinate_apache_reload + """) % context + ) + super(PHPController, self).commit() + + def get_fpm_config(self, webapp, context): + options = webapp.type_instance.get_options() + context.update({ + 'init_vars': webapp.type_instance.get_php_init_vars(merge=self.MERGE), + 'max_children': options.get('processes', settings.WEBAPPS_FPM_DEFAULT_MAX_CHILDREN), + 'request_terminate_timeout': options.get('timeout', False), + }) + context['fpm_listen'] = webapp.type_instance.FPM_LISTEN % context + fpm_config = Template(textwrap.dedent("""\ + ;; {{ banner }} + [{{ user }}-{{app_name}}] + user = {{ user }} + group = {{ group }} + + listen = {{ fpm_listen | safe }} + listen.owner = {{ user }} + listen.group = {{ group }} + + pm = ondemand + pm.max_requests = {{ max_requests }} + pm.max_children = {{ max_children }} + {% if request_terminate_timeout %} + request_terminate_timeout = {{ request_terminate_timeout }}{% endif %} + {% for name, value in init_vars.items %} + php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %} + """ + )) + return fpm_config.render(Context(context)) + + def get_fcgid_wrapper(self, webapp, context): + opt = webapp.type_instance + # Format PHP init vars + init_vars = opt.get_php_init_vars(merge=self.MERGE) + if init_vars: + init_vars = [ " \\\n -d %s='%s'" % (k, v.replace("'", '"')) for k,v in init_vars.items() ] + init_vars = ''.join(init_vars) + context.update({ + 'php_binary_path': os.path.normpath(settings.WEBAPPS_PHP_CGI_BINARY_PATH % context), + 'php_rc': os.path.normpath(settings.WEBAPPS_PHP_CGI_RC_DIR % context), + 'php_ini_scan': os.path.normpath(settings.WEBAPPS_PHP_CGI_INI_SCAN_DIR % context), + 'php_init_vars': init_vars, + }) + context['php_binary'] = os.path.basename(context['php_binary_path']) + return textwrap.dedent("""\ + #!/bin/sh + # %(banner)s + export PHPRC=%(php_rc)s + export PHP_INI_SCAN_DIR=%(php_ini_scan)s + export PHP_FCGI_MAX_REQUESTS=%(max_requests)s + exec %(php_binary_path)s%(php_init_vars)s""") % context + + def get_fcgid_cmd_options(self, webapp, context): + options = webapp.type_instance.get_options() + maps = OrderedDict( + MaxProcesses=options.get('processes', None), + IOTimeout=options.get('timeout', None), + ) + cmd_options = [] + for directive, value in maps.items(): + if value: + cmd_options.append( + "%s %s" % (directive, value.replace("'", '"')) + ) + if cmd_options: + head = ( + '# %(banner)s\n' + 'FcgidCmdOptions %(wrapper_path)s' + ) % context + cmd_options.insert(0, head) + return ' \\\n '.join(cmd_options) + + def update_fcgid_context(self, webapp, context): + wrapper_path = settings.WEBAPPS_FCGID_WRAPPER_PATH % context + context.update({ + 'wrapper': self.get_fcgid_wrapper(webapp, context), + 'wrapper_path': wrapper_path, + 'wrapper_dir': os.path.dirname(wrapper_path), + }) + context.update({ + 'cmd_options': self.get_fcgid_cmd_options(webapp, context), + 'cmd_options_path': settings.WEBAPPS_FCGID_CMD_OPTIONS_PATH % context, + }) + return context + + def update_fpm_context(self, webapp, context): + context.update({ + 'fpm_config': self.get_fpm_config(webapp, context), + 'fpm_path': settings.WEBAPPS_PHPFPM_POOL_PATH % context, + }) + return context + + def get_context(self, webapp): + context = super().get_context(webapp) + context.update({ + 'max_requests': settings.WEBAPPS_PHP_MAX_REQUESTS, + }) + self.update_fpm_context(webapp, context) + self.update_fcgid_context(webapp, context) + return context diff --git a/orchestra/contrib/webapps/backends/python.py b/orchestra/contrib/webapps/backends/python.py new file mode 100644 index 00000000..63013d5e --- /dev/null +++ b/orchestra/contrib/webapps/backends/python.py @@ -0,0 +1,86 @@ +import os +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace + +from . import WebAppServiceMixin +from .. import settings + + +class uWSGIPythonController(WebAppServiceMixin, ServiceController): + """ + Emperor mode + """ + verbose_name = _("Python uWSGI") + default_route_match = "webapp.type.endswith('python')" + doc_settings = (settings, ( + 'WEBAPPS_UWSGI_BASE_DIR', + 'WEBAPPS_PYTHON_MAX_REQUESTS', + 'WEBAPPS_PYTHON_DEFAULT_MAX_WORKERS', + 'WEBAPPS_PYTHON_DEFAULT_TIMEOUT', + )) + + def save(self, webapp): + context = self.get_context(webapp) + self.create_webapp_dir(context) + self.set_under_construction(context) + self.save_uwsgi(webapp, context) + + def delete(self, webapp): + context = self.get_context(webapp) + self.delete_uwsgi(webapp, context) + self.delete_webapp_dir(context) + + def save_uwsgi(self, webapp, context): + self.append("echo '%(uwsgi_config)s' > %(vassal_path)s" % context) + + def delete_uwsgi(self, webapp, context): + self.append("rm -f %(vassal_path)s" % context) + + def get_uwsgi_ini(self, context): + return textwrap.dedent("""\ + # %(banner)s + [uwsgi] + plugins = python{python_version_number} + chdir = {app_path} + module = {app_name}.wsgi + chmod-socket = 660 + stats = /run/uwsgi/%(deb-confnamespace)/%(deb-confname)/statsocket + vacuum = true + uid = {user} + gid = {group} + env = HOME={home} + harakiri = {timeout} + max-requests = {max_requests} + + cheaper-algo = spare + cheaper = 1 + workers = {workers} + cheaper-step = 1 + cheaper-overload = 5""" + ).format(context) + + def update_uwsgi_context(self, webapp, context): + context.update({ + 'uwsgi_ini': self.get_uwsgi_ini(context), + 'uwsgi_dir': settings.WEBAPPS_UWSGI_BASE_DIR, + 'vassal_path': os.path.join(settings.WEBAPPS_UWSGI_BASE_DIR, + 'vassals/%s' % context['app_name']), + }) + return context + + def get_context(self, webapp): + context = super(uWSGIPythonController, self).get_context(webapp) + options = webapp.get_options() + context.update({ + 'python_version': webapp.type_instance.get_python_version(), + 'python_version_number': webapp.type_instance.get_python_version_number(), + 'max_requests': settings.WEBAPPS_PYTHON_MAX_REQUESTS, + 'workers': options.get('processes', settings.WEBAPPS_PYTHON_DEFAULT_MAX_WORKERS), + 'timeout': options.get('timeout', settings.WEBAPPS_PYTHON_DEFAULT_TIMEOUT), + }) + self.update_uwsgi_context(webapp, context) + replace(context, "'", '"') + return context diff --git a/orchestra/contrib/webapps/backends/static.py b/orchestra/contrib/webapps/backends/static.py new file mode 100644 index 00000000..f424d505 --- /dev/null +++ b/orchestra/contrib/webapps/backends/static.py @@ -0,0 +1,23 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController + +from . import WebAppServiceMixin + + +class StaticController(WebAppServiceMixin, ServiceController): + """ + Static web pages. + Only creates the webapp dir and leaves the web server the decision to execute CGIs or not. + """ + verbose_name = _("Static") + default_route_match = "webapp.type == 'static'" + + def save(self, webapp): + context = self.get_context(webapp) + self.create_webapp_dir(context) + self.set_under_construction(context) + + def delete(self, webapp): + context = self.get_context(webapp) + self.delete_webapp_dir(context) diff --git a/orchestra/contrib/webapps/backends/symboliclink.py b/orchestra/contrib/webapps/backends/symboliclink.py new file mode 100644 index 00000000..2ba3fac2 --- /dev/null +++ b/orchestra/contrib/webapps/backends/symboliclink.py @@ -0,0 +1,35 @@ +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace + +from .php import PHPController + + +class SymbolicLinkController(PHPController, ServiceController): + """ + Same as PHPController but allows you to have the webapps on a directory diferent than the webapps dir. + """ + verbose_name = _("Symbolic link webapp") + model = 'webapps.WebApp' + default_route_match = "webapp.type == 'symbolic-link'" + + def create_webapp_dir(self, context): + self.append(textwrap.dedent("""\ + if [[ ! -e %(app_path)s ]]; then + ln -s '%(link_path)s' %(app_path)s + fi + chown -h %(user)s:%(group)s %(app_path)s + """) % context + ) + + def set_under_construction(self, context): + pass + + def get_context(self, webapp): + context = super(SymbolicLinkController, self).get_context(webapp) + context.update({ + 'link_path': webapp.data['path'], + }) + return replace(context, "'", '"') diff --git a/orchestra/contrib/webapps/backends/webalizer.py b/orchestra/contrib/webapps/backends/webalizer.py new file mode 100644 index 00000000..c9d35560 --- /dev/null +++ b/orchestra/contrib/webapps/backends/webalizer.py @@ -0,0 +1,22 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController + +from . import WebAppServiceMixin + + +# TODO DEPRECATE +class WebalizerAppController(WebAppServiceMixin, ServiceController): + """ + Needed for cleaning up webalizer main folder when webapp deleteion withou related contents + """ + verbose_name = _("Webalizer App") + default_route_match = "webapp.type == 'webalizer'" + + def save(self, webapp): + context = self.get_context(webapp) + self.create_webapp_dir(context) + + def delete(self, webapp): + context = self.get_context(webapp) + self.delete_webapp_dir(context) diff --git a/orchestra/contrib/webapps/backends/wordpress.py b/orchestra/contrib/webapps/backends/wordpress.py new file mode 100644 index 00000000..819be388 --- /dev/null +++ b/orchestra/contrib/webapps/backends/wordpress.py @@ -0,0 +1,148 @@ +import os +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController, replace + +from .. import settings + +from . import WebAppServiceMixin + + +# Based on https://github.com/mtomic/wordpress-install/blob/master/wpinstall.php +class WordPressController(WebAppServiceMixin, ServiceController): + """ + Installs the latest version of WordPress available on www.wordpress.org + It fully configures the wp-config.php (keys included) and sets up the database with initial admin password. + """ + verbose_name = _("Wordpress") + model = 'webapps.WebApp' + default_route_match = "webapp.type == 'wordpress-php'" + script_executable = '/usr/bin/php' + doc_settings = (settings, + ('WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST',) + ) + + def prepare(self): + self.append(textwrap.dedent("""\ + 1) { + die("App directory not empty."); + } + // Download and untar wordpress (with caching system) + shell_exec("mkdir -p %(app_path)s + # Prevent other backends from writting here + touch %(app_path)s/.lock + filename=\\$(wget https://wordpress.org/latest.tar.gz --server-response --spider --no-check-certificate 2>&1 \\ + | grep filename | cut -d'=' -f2) + mkdir -p %(cms_cache_dir)s + if [ ! -e %(cms_cache_dir)s/wordpress ] || [ \\$(basename \\$(readlink %(cms_cache_dir)s/wordpress) 2> /dev/null ) != \\$filename ]; then + wget https://wordpress.org/latest.tar.gz -O - --no-check-certificate \\ + | tee %(cms_cache_dir)s/\\$filename \\ + | tar -xzvf - -C %(app_path)s --strip-components=1 + rm -f %(cms_cache_dir)s/wordpress + ln -s %(cms_cache_dir)s/\\$filename %(cms_cache_dir)s/wordpress + else + tar -xzvf %(cms_cache_dir)s/wordpress -C %(app_path)s --strip-components=1 + fi + mkdir %(app_path)s/wp-content/uploads + chmod 750 %(app_path)s/wp-content/uploads + rm %(app_path)s/.lock + "); + + $config_file = file('%(app_path)s/' . 'wp-config-sample.php'); + $secret_keys = file_get_contents('https://api.wordpress.org/secret-key/1.1/salt/'); + $secret_keys = explode( "\\n", $secret_keys ); + foreach ( $secret_keys as $k => $v ) { + $secret_keys[$k] = substr( $v, 28, 64 ); + } + array_pop($secret_keys); + + // setup wordpress database and keys config + $config_file = str_replace('database_name_here', "%(db_name)s", $config_file); + $config_file = str_replace('username_here', "%(db_user)s", $config_file); + $config_file = str_replace('password_here', "%(password)s", $config_file); + $config_file = str_replace('localhost', "%(db_host)s", $config_file); + $config_file = str_replace("'AUTH_KEY', 'put your unique phrase here'", "'AUTH_KEY', '{$secret_keys[0]}'", $config_file); + $config_file = str_replace("'SECURE_AUTH_KEY', 'put your unique phrase here'", "'SECURE_AUTH_KEY', '{$secret_keys[1]}'", $config_file); + $config_file = str_replace("'LOGGED_IN_KEY', 'put your unique phrase here'", "'LOGGED_IN_KEY', '{$secret_keys[2]}'", $config_file); + $config_file = str_replace("'NONCE_KEY', 'put your unique phrase here'", "'NONCE_KEY', '{$secret_keys[3]}'", $config_file); + $config_file = str_replace("'AUTH_SALT', 'put your unique phrase here'", "'AUTH_SALT', '{$secret_keys[4]}'", $config_file); + $config_file = str_replace("'SECURE_AUTH_SALT', 'put your unique phrase here'", "'SECURE_AUTH_SALT', '{$secret_keys[5]}'", $config_file); + $config_file = str_replace("'LOGGED_IN_SALT', 'put your unique phrase here'", "'LOGGED_IN_SALT', '{$secret_keys[6]}'", $config_file); + $config_file = str_replace("'NONCE_SALT', 'put your unique phrase here'", "'NONCE_SALT', '{$secret_keys[7]}'", $config_file); + + if(file_exists('%(app_path)s/' .'wp-config.php')) { + unlink('%(app_path)s/' .'wp-config.php'); + } + + $fw = fopen('%(app_path)s/' . 'wp-config.php', 'a'); + foreach ( $config_file as $line_num => $line ) { + fwrite($fw, $line); + } + exc('chown -R %(user)s:%(group)s %(app_path)s'); + + // Run wordpress installation process + + define('WP_CONTENT_DIR', 'wp-content/'); + define('WP_LANG_DIR', WP_CONTENT_DIR . '/languages' ); + define('WP_USE_THEMES', true); + define('DB_NAME', "%(db_name)s"); + define('DB_USER', "%(db_user)s"); + define('DB_PASSWORD', "%(password)s"); + define('DB_HOST', "%(db_host)s"); + + $_GET['step'] = 2; + $_POST['weblog_title'] = "%(title)s"; + $_POST['user_name'] = "admin"; + $_POST['admin_email'] = "%(email)s"; + $_POST['blog_public'] = true; + $_POST['admin_password'] = "%(password)s"; + $_POST['admin_password2'] = "%(password)s"; + + ob_start(); + require_once('%(app_path)s/wp-admin/install.php'); + $response = ob_get_contents(); + ob_end_clean(); + if (strpos($response, '

    Success!

    ') === false) { + echo "Error has occured during installation\\n"; + echo $msg; + exit(1); + }""") % context + ) + + def commit(self): + self.append('?>') + + def delete(self, webapp): + context = self.get_context(webapp) + self.append("exc('rm -rf %(app_path)s');" % context) + + def get_context(self, webapp): + context = super(WordPressController, self).get_context(webapp) + context.update({ + 'db_name': webapp.data['db_name'], + 'db_user': webapp.data['db_user'], + 'password': webapp.data['password'], + 'db_host': settings.WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST, + 'email': webapp.account.email, + 'title': "%s blog's" % webapp.account.get_full_name(), + 'cms_cache_dir': os.path.normpath(settings.WEBAPPS_CMS_CACHE_DIR) + }) + return replace(context, '"', "'") diff --git a/orchestra/contrib/webapps/fields.py b/orchestra/contrib/webapps/fields.py new file mode 100644 index 00000000..d430a414 --- /dev/null +++ b/orchestra/contrib/webapps/fields.py @@ -0,0 +1,28 @@ +from django.contrib.contenttypes.fields import GenericRelation +from django.db import DEFAULT_DB_ALIAS + + +class VirtualDatabaseRelation(GenericRelation): + """ Delete related databases if any """ + def bulk_related_objects(self, objs, using=DEFAULT_DB_ALIAS): + pks = [] + for obj in objs: + db_id = obj.data.get('db_id') + if db_id: + pks.append(db_id) + if not pks: + return [] + return self.remote_field.model._base_manager.db_manager(using).filter(pk__in=pks) + + +class VirtualDatabaseUserRelation(GenericRelation): + """ Delete related databases if any """ + def bulk_related_objects(self, objs, using=DEFAULT_DB_ALIAS): + pks = [] + for obj in objs: + db_id = obj.data.get('db_user_id') + if db_id: + pks.append(db_id) + if not pks: + return [] + return self.remote_field.model._base_manager.db_manager(using).filter(pk__in=pks) diff --git a/orchestra/contrib/webapps/filters.py b/orchestra/contrib/webapps/filters.py new file mode 100644 index 00000000..794217bf --- /dev/null +++ b/orchestra/contrib/webapps/filters.py @@ -0,0 +1,51 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + +from .types import AppType + + +class HasWebsiteListFilter(SimpleListFilter): + title = _("website") + parameter_name = 'has_website' + + def lookups(self, request, model_admin): + return ( + ('True', _("True")), + ('False', _("False")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(content__isnull=False) + elif self.value() == 'False': + return queryset.filter(content__isnull=True) + return queryset + + +class DetailListFilter(SimpleListFilter): + title = _("detail") + parameter_name = 'detail' + + def lookups(self, request, model_admin): + ret = set([('empty', _("Empty"))]) + lookup_map = {} + for apptype in AppType.get_plugins(): + for field, values in apptype.get_detail_lookups().items(): + for value in values: + lookup_map[value[0]] = field + ret.add(value) + self.lookup_map = lookup_map + return sorted(list(ret), key=lambda e: e[1]) + + def queryset(self, request, queryset): + value = self.value() + if value: + if value == 'empty': + return queryset.filter(data={}) + try: + field = self.lookup_map[value] + except KeyError: + return queryset + else: + return queryset.filter(data__contains='"%s":"%s"' % (field, value)) + return queryset diff --git a/orchestra/contrib/webapps/models.py b/orchestra/contrib/webapps/models.py new file mode 100644 index 00000000..21f07979 --- /dev/null +++ b/orchestra/contrib/webapps/models.py @@ -0,0 +1,125 @@ +import os +from collections import OrderedDict + +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ +from jsonfield import JSONField + +from orchestra.core import validators + +from . import settings +from .fields import VirtualDatabaseRelation, VirtualDatabaseUserRelation +from .options import AppOption +from .types import AppType + + +class WebApp(models.Model): + """ Represents a web application """ + name = models.CharField(_("name"), max_length=128, validators=[validators.validate_name], + help_text=_("The app will be installed in %s") % settings.WEBAPPS_BASE_DIR) + type = models.CharField(_("type"), max_length=32, choices=AppType.get_choices()) + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='webapps') + data = JSONField(_("data"), blank=True, default={}, + help_text=_("Extra information dependent of each service.")) + target_server = models.ForeignKey('orchestration.Server', on_delete=models.CASCADE, + verbose_name=_("Target Server"), related_name='webapps') + comments = models.TextField(default="", blank=True) + + # CMS webapps usually need a database and dbuser, with these virtual fields we tell the ORM to delete them + databases = VirtualDatabaseRelation('databases.Database') + databaseusers = VirtualDatabaseUserRelation('databases.DatabaseUser') + + class Meta: + unique_together = ('name', 'account') + verbose_name = _("Web App") + verbose_name_plural = _("Web Apps") + + def __str__(self): + return self.name + + def get_description(self): + return self.get_type_display() + + @cached_property + def type_class(self): + return AppType.get(self.type) + + @cached_property + def type_instance(self): + """ Per request lived type_instance """ + return self.type_class(self) + + def clean(self): + apptype = self.type_instance + apptype.validate() + a = apptype.clean_data() + self.data = apptype.clean_data() + + def get_options(self, **kwargs): + options = OrderedDict() + qs = WebAppOption.objects.filter(**kwargs) + for name, value in qs.values_list('name', 'value').order_by('name'): + if name in options: + if AppOption.get(name).comma_separated: + options[name] = options[name].rstrip(',') + ',' + value.lstrip(',') + else: + options[name] = max(options[name], value) + else: + options[name] = value + return options + + def get_directive(self): + return self.type_instance.get_directive() + + def get_base_path(self): + context = { + 'home': self.get_user().get_home(), + 'app_name': self.name, + } + return settings.WEBAPPS_BASE_DIR % context + + def get_path(self): + path = self.get_base_path() + public_root = self.options.filter(name='public-root').first() + if public_root: + path = os.path.join(path, public_root.value) + return os.path.normpath(path.replace('//', '/')) + + def get_user(self): + return self.account.main_systemuser + + def get_username(self): + return self.get_user().username + + def get_groupname(self): + return self.get_username() + + +class WebAppOption(models.Model): + webapp = models.ForeignKey(WebApp, on_delete=models.CASCADE, + verbose_name=_("Web application"), related_name='options') + name = models.CharField(_("name"), max_length=128, + choices=AppType.get_group_options_choices()) + value = models.CharField(_("value"), max_length=256) + + class Meta: + unique_together = ('webapp', 'name') + verbose_name = _("option") + verbose_name_plural = _("options") + + def __str__(self): + return self.name + + @cached_property + def option_class(self): + return AppOption.get(self.name) + + @cached_property + def option_instance(self): + """ Per request lived option instance """ + return self.option_class(self) + + def clean(self): + self.option_instance.validate() diff --git a/orchestra/contrib/webapps/options.py b/orchestra/contrib/webapps/options.py new file mode 100644 index 00000000..66a9a373 --- /dev/null +++ b/orchestra/contrib/webapps/options.py @@ -0,0 +1,383 @@ +import os +import re +from functools import lru_cache + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra import plugins +from orchestra.utils.python import import_class + +from . import settings + + +class AppOption(plugins.Plugin, metaclass=plugins.PluginMount): + PHP = 'PHP' + PROCESS = 'Process' + FILESYSTEM = 'FileSystem' + + help_text = "" + group = None + comma_separated = False + + @classmethod + @lru_cache() + def get_plugins(cls, all=False): + if all: + plugins = super().get_plugins() + else: + plugins = [] + for cls in settings.WEBAPPS_ENABLED_OPTIONS: + plugins.append(import_class(cls)) + return plugins + + @classmethod + @lru_cache() + def get_option_groups(cls): + groups = {} + for opt in cls.get_plugins(): + try: + groups[opt.group].append(opt) + except KeyError: + groups[opt.group] = [opt] + return groups + + def validate(self): + if self.regex and not re.match(self.regex, self.instance.value): + raise ValidationError({ + 'value': ValidationError(_("'%(value)s' does not match %(regex)s."), + params={ + 'value': self.instance.value, + 'regex': self.regex + }), + }) + + +class PHPAppOption(AppOption): + deprecated = None + group = AppOption.PHP + abstract = True + + def validate(self): + super().validate() + if self.deprecated: + php_version = self.instance.webapp.type_instance.get_php_version_number() + if php_version and self.deprecated and float(php_version) > self.deprecated: + raise ValidationError( + _("This option is deprecated since PHP version %s.") % self.deprecated + ) + + +class PublicRoot(AppOption): + name = 'public-root' + verbose_name = _("Public root") + help_text = _("Document root relative to webapps/<webapp>/") + regex = r'[^ ]+' + group = AppOption.FILESYSTEM + + def validate(self): + super().validate() + base_path = self.instance.webapp.get_base_path() + path = os.path.join(base_path, self.instance.value) + if not os.path.abspath(path).startswith(base_path): + raise ValidationError( + _("Public root path '%s' outside of webapp base path '%s'") % (path, base_path) + ) + + +class Timeout(AppOption): + name = 'timeout' + # FCGID FcgidIOTimeout + # FPM pm.request_terminate_timeout + # PHP max_execution_time ini + verbose_name = _("Process timeout") + help_text = _("Maximum time in seconds allowed for a request to complete (a number between 0 and 999).
    " + "Also sets max_request_time when php-cgi is used.") + regex = r'^[0-9]{1,3}$' + group = AppOption.PROCESS + + +class Processes(AppOption): + name = 'processes' + # FCGID MaxProcesses + # FPM pm.max_children + verbose_name = _("Number of processes") + help_text = _("Maximum number of children that can be alive at the same time (a number between 0 and 99).") + regex = r'^[0-9]{1,3}$' + group = AppOption.PROCESS + + +class PHPEnableFunctions(PHPAppOption): + name = 'enable_functions' + verbose_name = _("Enable functions") + help_text = '%s' % ',
    '.join([ + ','.join(settings.WEBAPPS_PHP_DISABLED_FUNCTIONS[i:i+10]) + for i in range(0, len(settings.WEBAPPS_PHP_DISABLED_FUNCTIONS), 10) + ]) + regex = r'^[\w\.,-]+$' + comma_separated = True + + def validate(self): + # Clean value removing spaces + self.instance.value = self.instance.value.replace(' ', '') + super().validate() + + +class PHPDisableFunctions(PHPAppOption): + name = 'disable_functions' + verbose_name = _("Disable functions") + help_text = _("This directive allows you to disable certain functions for security reasons. " + "It takes on a comma-delimited list of function names. disable_functions is not " + "affected by Safe Mode. Default disabled fuctions include:
    " + "%s") % ',
    '.join([ + ','.join(settings.WEBAPPS_PHP_DISABLED_FUNCTIONS[i:i+10]) + for i in range(0, len(settings.WEBAPPS_PHP_DISABLED_FUNCTIONS), 10) + ]) + regex = r'^[\w\.,-]+$' + comma_separated = True + + def validate(self): + # Clean value removing spaces + self.instance.value = self.instance.value.replace(' ', '') + super().validate() + + +class PHPAllowURLInclude(PHPAppOption): + name = 'allow_url_include' + verbose_name = _("Allow URL include") + help_text = _("Allows the use of URL-aware fopen wrappers with include, include_once, require, " + "require_once (On or Off).") + regex = r'^(On|Off|on|off)$' + + +class PHPAllowURLFopen(PHPAppOption): + name = 'allow_url_fopen' + verbose_name = _("Allow URL fopen") + help_text = _("Enables the URL-aware fopen wrappers that enable accessing URL object like files (On or Off).") + regex = r'^(On|Off|on|off)$' + + +class PHPAutoAppendFile(PHPAppOption): + name = 'auto_append_file' + verbose_name = _("Auto append file") + help_text = _("Specifies the name of a file that is automatically parsed after the main file.") + regex = r'^[\w\.,-/]+$' + + +class PHPAutoPrependFile(PHPAppOption): + name = 'auto_prepend_file' + verbose_name = _("Auto prepend file") + help_text = _("Specifies the name of a file that is automatically parsed before the main file.") + regex = r'^[\w\.,-/]+$' + + +class PHPDateTimeZone(PHPAppOption): + name = 'date.timezone' + verbose_name = _("date.timezone") + help_text = _("Sets the default timezone used by all date/time functions (Timezone string 'Europe/London').") + regex = r'^\w+/\w+$' + + +class PHPDefaultSocketTimeout(PHPAppOption): + name = 'default_socket_timeout' + verbose_name = _("Default socket timeout") + help_text = _("Number between 0 and 999.") + regex = r'^[0-9]{1,3}$' + + +class PHPDisplayErrors(PHPAppOption): + name = 'display_errors' + verbose_name = _("Display errors") + help_text = _("Determines whether errors should be printed to the screen as part of the output or " + "if they should be hidden from the user (On or Off).") + regex = r'^(On|Off|on|off)$' + + +class PHPExtension(PHPAppOption): + name = 'extension' + verbose_name = _("Extension") + regex = r'^[^ ]+$' + + +class PHPIncludePath(PHPAppOption): + name = 'include_path' + verbose_name = _("Include path") + regex = r'^[^ ]+$' + +class PHPOpenBasedir(PHPAppOption): + name = 'open_basedir' + verbose_name = _("Open basedir") + regex = r'^[^ ]+$' + +class PHPMagicQuotesGPC(PHPAppOption): + name = 'magic_quotes_gpc' + verbose_name = _("Magic quotes GPC") + help_text = _("Sets the magic_quotes state for GPC (Get/Post/Cookie) operations (On or Off) " + "DEPRECATED as of PHP 5.3.0.") + regex = r'^(On|Off|on|off)$' + deprecated = 5.3 + + +class PHPMagicQuotesRuntime(PHPAppOption): + name = 'magic_quotes_runtime' + verbose_name = _("Magic quotes runtime") + help_text = _("Functions that return data from any sort of external source will have quotes escaped " + "with a backslash (On or Off) DEPRECATED as of PHP 5.3.0.") + regex = r'^(On|Off|on|off)$' + deprecated = 5.3 + + +class PHPMaginQuotesSybase(PHPAppOption): + name = 'magic_quotes_sybase' + verbose_name = _("Magic quotes sybase") + help_text = _("Single-quote is escaped with a single-quote instead of a backslash (On or Off).") + regex = r'^(On|Off|on|off)$' + + +class PHPMaxInputTime(PHPAppOption): + name = 'max_input_time' + verbose_name = _("Max input time") + help_text = _("Maximum time in seconds a script is allowed to parse input data, like POST and GET " + "(Integer between 0 and 999).") + regex = r'^[0-9]{1,3}$' + + +class PHPMaxInputVars(PHPAppOption): + name = 'max_input_vars' + verbose_name = _("Max input vars") + help_text = _("How many input variables may be accepted (limit is applied to $_GET, $_POST " + "and $_COOKIE superglobal separately) (Integer between 0 and 9999).") + regex = r'^[0-9]{1,4}$' + + +class PHPMemoryLimit(PHPAppOption): + name = 'memory_limit' + verbose_name = _("Memory limit") + help_text = _("This sets the maximum amount of memory in bytes that a script is allowed to allocate " + "(Value between 0M and 999M).") + regex = r'^[0-9]{1,3}M$' + + +class PHPMySQLConnectTimeout(PHPAppOption): + name = 'mysql.connect_timeout' + verbose_name = _("Mysql connect timeout") + help_text = _("Number between 0 and 999.") + regex = r'^([0-9]){1,3}$' + + +class PHPOutputBuffering(PHPAppOption): + name = 'output_buffering' + verbose_name = _("Output buffering") + help_text = _("Turn on output buffering (On or Off).") + regex = r'^(On|Off|on|off)$' + + +class PHPRegisterGlobals(PHPAppOption): + name = 'register_globals' + verbose_name = _("Register globals") + help_text = _("Whether or not to register the EGPCS (Environment, GET, POST, Cookie, Server) " + "variables as global variables (On or Off).") + regex = r'^(On|Off|on|off)$' + + +class PHPPostMaxSize(PHPAppOption): + name = 'post_max_size' + verbose_name = _("Post max size") + help_text = _("Sets max size of post data allowed (Value between 0M and 999M).") + regex = r'^[0-9]{1,3}M$' + + +class PHPSendmailPath(PHPAppOption): + name = 'sendmail_path' + verbose_name = _("Sendmail path") + help_text = _("Where the sendmail program can be found.") + regex = r'^[^ ]+$' + + +class PHPSessionBugCompatWarn(PHPAppOption): + name = 'session.bug_compat_warn' + verbose_name = _("Session bug compat warning") + help_text = _("Enables an PHP bug on session initialization for legacy behaviour (On or Off).") + regex = r'^(On|Off|on|off)$' + + +class PHPSessionAutoStart(PHPAppOption): + name = 'session.auto_start' + verbose_name = _("Session auto start") + help_text = _("Specifies whether the session module starts a session automatically on request " + "startup (On or Off).") + regex = r'^(On|Off|on|off)$' + group = AppOption.PHP + + +class PHPSafeMode(PHPAppOption): + name = 'safe_mode' + verbose_name = _("Safe mode") + help_text = _("Whether to enable PHP's safe mode (On or Off) DEPRECATED as of PHP 5.3.0") + regex = r'^(On|Off|on|off)$' + deprecated=5.3 + + +class PHPSuhosinPostMaxVars(PHPAppOption): + name = 'suhosin.post.max_vars' + verbose_name = _("Suhosin POST max vars") + help_text = _("Number between 0 and 9999.") + regex = r'^[0-9]{1,4}$' + + +class PHPSuhosinGetMaxVars(PHPAppOption): + name = 'suhosin.get.max_vars' + verbose_name = _("Suhosin GET max vars") + help_text = _("Number between 0 and 9999.") + regex = r'^[0-9]{1,4}$' + + +class PHPSuhosinRequestMaxVars(PHPAppOption): + name = 'suhosin.request.max_vars' + verbose_name = _("Suhosin request max vars") + help_text = _("Number between 0 and 9999.") + regex = r'^[0-9]{1,4}$' + + +class PHPSuhosinSessionEncrypt(PHPAppOption): + name = 'suhosin.session.encrypt' + verbose_name = _("Suhosin session encrypt") + help_text = _("On or Off") + regex = r'^(On|Off|on|off)$' + + +class PHPSuhosinSimulation(PHPAppOption): + name = 'suhosin.simulation' + verbose_name = _("Suhosin simulation") + help_text = _("On or Off") + regex = r'^(On|Off|on|off)$' + + +class PHPSuhosinExecutorIncludeWhitelist(PHPAppOption): + name = 'suhosin.executor.include.whitelist' + verbose_name = _("Suhosin executor include whitelist") + regex = r'.*$' + + +class PHPUploadMaxFileSize(PHPAppOption): + name = 'upload_max_filesize' + verbose_name = _("Upload max filesize") + help_text = _("Value between 0M and 999M.") + regex = r'^[0-9]{1,3}M$' + + +class PHPUploadTmpDir(PHPAppOption): + name = 'upload_tmp_dir' + verbose_name = _("Upload tmp dir") + help_text = _("The temporary directory used for storing files when doing file upload. " + "Must be writable by whatever user PHP is running as. " + "If not specified PHP will use the system's default.
    " + "If the directory specified here is not writable, PHP falls back to the " + "system default temporary directory. If open_basedir is on, then the system " + "default directory must be allowed for an upload to succeed.") + regex = r'.*$' + +class PHPZendExtension(PHPAppOption): + name = 'zend_extension' + verbose_name = _("Zend extension") + regex = r'^[^ ]+$' diff --git a/orchestra/contrib/webapps/serializers.py b/orchestra/contrib/webapps/serializers.py new file mode 100644 index 00000000..abfe9dc6 --- /dev/null +++ b/orchestra/contrib/webapps/serializers.py @@ -0,0 +1,69 @@ +from django.db import models +from rest_framework import serializers + +from orchestra.api.serializers import HyperlinkedModelSerializer +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .models import WebApp, WebAppOption + + +class OptionSerializer(serializers.ModelSerializer): + class Meta: + model = WebAppOption + fields = ('name', 'value') + + def to_representation(self, instance): + return {prop.name: prop.value for prop in instance.all()} + + def to_internal_value(self, data): + return data + + +class DataField(serializers.Field): + def to_representation(self, data): + return data + + +class WebAppSerializer(AccountSerializerMixin, HyperlinkedModelSerializer): + options = OptionSerializer(required=False) + data = DataField() + + class Meta: + model = WebApp + fields = ('url', 'id', 'name', 'type', 'options', 'data',) + postonly_fields = ('name', 'type') + + def __init__(self, *args, **kwargs): + super(WebAppSerializer, self).__init__(*args, **kwargs) + if isinstance(self.instance, models.Model): + type_serializer = self.instance.type_instance.serializer + if type_serializer: + self.fields['data'] = type_serializer() + + def create(self, validated_data): + options_data = validated_data.pop('options') + webapp = super(WebAppSerializer, self).create(validated_data) + for key, value in options_data.items(): + WebAppOption.objects.create(webapp=webapp, name=key, value=value) + return webap + + def update(self, instance, validated_data): + options_data = validated_data.pop('options') + instance = super(WebAppSerializer, self).update(instance, validated_data) + existing = {} + for obj in instance.options.all(): + existing[obj.name] = obj + posted = set() + for key, value in options_data.items(): + posted.add(key) + try: + option = existing[key] + except KeyError: + option = instance.options.create(name=key, value=value) + else: + if option.value != value: + option.value = value + option.save(update_fields=('value',)) + for to_delete in set(existing.keys())-posted: + existing[to_delete].delete() + return instance diff --git a/orchestra/contrib/webapps/settings.py b/orchestra/contrib/webapps/settings.py new file mode 100644 index 00000000..19471c59 --- /dev/null +++ b/orchestra/contrib/webapps/settings.py @@ -0,0 +1,284 @@ +from orchestra.contrib.settings import Setting +from orchestra.settings import ORCHESTRA_BASE_DOMAIN + +from .. import webapps + + +_names = ('home', 'user', 'user_id', 'group', 'app_type', 'app_name', 'app_type', 'app_id', 'account_id') +_php_names = _names + ('php_version', 'php_version_number', 'php_version_int') +_python_names = _names + ('python_version', 'python_version_number',) + + +WEBAPPS_BASE_DIR = Setting('WEBAPPS_BASE_DIR', + '%(home)s/webapps/%(app_name)s', + help_text="Available fromat names: %s" % ', '.join(_names), + validators=[Setting.string_format_validator(_names)], +) + + +WEBAPPS_FPM_LISTEN = Setting('WEBAPPS_FPM_LISTEN', + '127.0.0.1:5%(app_id)04d', + help_text=("TCP socket example: 127.0.0.1:5%(app_id)04d
    " + "UDS example: /var/lib/php/sockets/%(user)s-%(app_name)s.sock
    " + "Merged TCP example: 127.0.0.1:%(php_version_int)02d%(account_id)03d
    " + "Merged UDS example: /var/lib/php/sockets/%(user)s-%(php_version)s.sock
    " + "Available fromat names: {}").format(', '.join(_php_names)), + validators=[Setting.string_format_validator(_php_names)], +) + + +WEBAPPS_FPM_DEFAULT_MAX_CHILDREN = Setting('WEBAPPS_FPM_DEFAULT_MAX_CHILDREN', + 3 +) + + +WEBAPPS_PHPFPM_POOL_PATH = Setting('WEBAPPS_PHPFPM_POOL_PATH', + '/etc/php%(php_version_number)s/fpm/pool.d/%(user)s-%(app_name)s.conf', + help_text="Available fromat names: %s" % ', '.join(_php_names), + validators=[Setting.string_format_validator(_php_names)], +) + +WEBAPPS_PHPFPM_RELOAD_POOL = Setting('WEBAPPS_PHPFPM_RELOAD_POOL', + 'service php5-fpm reload' +) + + +WEBAPPS_FCGID_WRAPPER_PATH = Setting('WEBAPPS_FCGID_WRAPPER_PATH', + '/home/httpd/fcgi-bin.d/%(user)s/%(app_name)s-wrapper', + validators=[Setting.string_format_validator(_php_names)], + help_text=("Inside SuExec Document root.
    " + "Make sure all account wrappers are in the same DIR.
    " + "Available fromat names: %s") % ', '.join(_php_names), +) + + +WEBAPPS_FCGID_CMD_OPTIONS_PATH = Setting('WEBAPPS_FCGID_CMD_OPTIONS_PATH', + '/etc/apache2/fcgid-conf/%(user)s-%(app_name)s.conf', + validators=[Setting.string_format_validator(_php_names)], + help_text="Loaded by Apache. Available fromat names: %s" % ', '.join(_php_names), +) + + +WEBAPPS_PHP_MAX_REQUESTS = Setting('WEBAPPS_PHP_MAX_REQUESTS', + 400, + help_text='Greater or equal to your FcgidMaxRequestsPerProcess' +) + + +WEBAPPS_PHP_ERROR_LOG_PATH = Setting('WEBAPPS_PHP_ERROR_LOG_PATH', + '' +) + + +WEBAPPS_MERGE_PHP_WEBAPPS = Setting('WEBAPPS_MERGE_PHP_WEBAPPS', + False, + help_text=("Combine all fcgid-wrappers/fpm-pools into one per account-php_version " + "to better control num processes per account and save memory") +) + +WEBAPPS_TYPES = Setting('WEBAPPS_TYPES', ( + 'orchestra.contrib.webapps.types.php.PHPApp', + 'orchestra.contrib.webapps.types.misc.StaticApp', + 'orchestra.contrib.webapps.types.misc.WebalizerApp', + 'orchestra.contrib.webapps.types.misc.SymbolicLinkApp', + 'orchestra.contrib.webapps.types.wordpress.WordPressApp', + 'orchestra.contrib.webapps.types.moodle.MoodleApp', + 'orchestra.contrib.webapps.types.python.PythonApp', + ), + # lazy loading + choices=lambda : ((t.get_class_path(), t.get_class_path()) for t in webapps.types.AppType.get_plugins(all=True)), + multiple=True, +) + + +WEBAPPS_PHP_VERSIONS = Setting('WEBAPPS_PHP_VERSIONS', ( + ('5.6-fpm', 'PHP 5.6 FPM'), + ('5.6-cgi', 'PHP 5.6 FCGID'), + ('5.4-fpm', 'PHP 5.4 FPM'), + ('5.4-cgi', 'PHP 5.4 FCGID'), + ('5.3-cgi', 'PHP 5.3 FCGID'), + ('5.2-cgi', 'PHP 5.2 FCGID'), + ('4-cgi', 'PHP 4 FCGID'), + ('7-fpm', 'PHP 7 FPM') + ), + help_text="Execution modle choose by ending -fpm or -cgi.", + validators=[Setting.validate_choices] +) + + +WEBAPPS_DEFAULT_PHP_VERSION = Setting('WEBAPPS_DEFAULT_PHP_VERSION', + '5.6-fpm', + choices=WEBAPPS_PHP_VERSIONS +) + + +WEBAPPS_PHP_CGI_BINARY_PATH = Setting('WEBAPPS_PHP_CGI_BINARY_PATH', + '/usr/bin/php%(php_version_number)s-cgi', + help_text="Path of the cgi binary used by fcgid. Available fromat names: %s" % ', '.join(_php_names), + validators=[Setting.string_format_validator(_php_names)], +) + + +WEBAPPS_PHP_CGI_RC_DIR = Setting('WEBAPPS_PHP_CGI_RC_DIR', + '/etc/php%(php_version_number)s/cgi/', + help_text="Path to php.ini. Available fromat names: %s" % ', '.join(_php_names), + validators=[Setting.string_format_validator(_php_names)], +) + + +WEBAPPS_PHP_CGI_INI_SCAN_DIR = Setting('WEBAPPS_PHP_CGI_INI_SCAN_DIR', + '/etc/php%(php_version_number)s/cgi/conf.d', + help_text="Available fromat names: %s" % ', '.join(_php_names), + validators=[Setting.string_format_validator(_php_names)], +) + + +WEBAPPS_PYTHON_VERSIONS = Setting('WEBAPPS_PYTHON_VERSIONS', + ( + ('3.4-uwsgi', 'Python 3.4 uWSGI'), + ('2.7-uwsgi', 'Python 2.7 uWSGI'), + ), + validators=[Setting.validate_choices] +) + + +WEBAPPS_DEFAULT_PYTHON_VERSION = Setting('WEBAPPS_DEFAULT_PYTHON_VERSION', + '3.4-uwsgi', + choices=WEBAPPS_PYTHON_VERSIONS + +) + + +WEBAPPS_UWSGI_SOCKET = Setting('WEBAPPS_UWSGI_SOCKET', + '/var/run/uwsgi/app/%(app_name)s/socket', + help_text="Available fromat names: %s" % ', '.join(_python_names), + validators=[Setting.string_format_validator(_python_names)], +) + + +WEBAPPS_UWSGI_BASE_DIR = Setting('WEBAPPS_UWSGI_BASE_DIR', + '/etc/uwsgi/' +) + + +WEBAPPS_PYTHON_MAX_REQUESTS = Setting('WEBAPPS_PYTHON_MAX_REQUESTS', + 500 +) + + +WEBAPPS_PYTHON_DEFAULT_MAX_WORKERS = Setting('WEBAPPS_PYTHON_DEFAULT_MAX_WORKERS', + 3 +) + + +WEBAPPS_PYTHON_DEFAULT_TIMEOUT = Setting('WEBAPPS_PYTHON_DEFAULT_TIMEOUT', + 30 +) + + +WEBAPPS_UNDER_CONSTRUCTION_PATH = Setting('WEBAPPS_UNDER_CONSTRUCTION_PATH', '', + help_text=("Server-side path where a under construction stock page is " + "'/var/www/undercontruction/index.html'") +) + + +#WEBAPPS_TYPES_OVERRIDE = getattr(settings, 'WEBAPPS_TYPES_OVERRIDE', {}) +#for webapp_type, value in WEBAPPS_TYPES_OVERRIDE.items(): +# if value is None: +# WEBAPPS_TYPES.pop(webapp_type, None) +# else: +# WEBAPPS_TYPES[webapp_type] = value + + +WEBAPPS_PHP_DISABLED_FUNCTIONS = Setting('WEBAPPS_PHP_DISABLED_FUNCTION', ( + 'exec', + 'passthru', + 'shell_exec', + 'system', + 'proc_open', + 'popen', + 'curl_multi_exec', + 'show_source', + 'pcntl_exec', + 'proc_close', + 'proc_get_status', + 'proc_nice', + 'proc_terminate', + 'ini_alter', + 'virtual', + 'openlog', + 'escapeshellcmd', + 'escapeshellarg', + 'dl', + 'fsockopen', + 'pfsockopen', + 'stream_socket_client', + # Used for spamming + 'getmxrr', + # Used in some php shells + 'str_rot13', +)) + + +WEBAPPS_ENABLED_OPTIONS = Setting('WEBAPPS_ENABLED_OPTIONS', ( + 'orchestra.contrib.webapps.options.PublicRoot', + 'orchestra.contrib.webapps.options.Timeout', + 'orchestra.contrib.webapps.options.Processes', + 'orchestra.contrib.webapps.options.PHPEnableFunctions', + 'orchestra.contrib.webapps.options.PHPDisableFunctions', + 'orchestra.contrib.webapps.options.PHPAllowURLInclude', + 'orchestra.contrib.webapps.options.PHPAllowURLFopen', + 'orchestra.contrib.webapps.options.PHPAutoAppendFile', + 'orchestra.contrib.webapps.options.PHPAutoPrependFile', + 'orchestra.contrib.webapps.options.PHPDateTimeZone', + 'orchestra.contrib.webapps.options.PHPDefaultSocketTimeout', + 'orchestra.contrib.webapps.options.PHPDisplayErrors', + 'orchestra.contrib.webapps.options.PHPExtension', + 'orchestra.contrib.webapps.options.PHPIncludePath', + 'orchestra.contrib.webapps.options.PHPOpenBasedir', + 'orchestra.contrib.webapps.options.PHPMagicQuotesGPC', + 'orchestra.contrib.webapps.options.PHPMagicQuotesRuntime', + 'orchestra.contrib.webapps.options.PHPMaginQuotesSybase', + 'orchestra.contrib.webapps.options.PHPMaxInputTime', + 'orchestra.contrib.webapps.options.PHPMaxInputVars', + 'orchestra.contrib.webapps.options.PHPMemoryLimit', + 'orchestra.contrib.webapps.options.PHPMySQLConnectTimeout', + 'orchestra.contrib.webapps.options.PHPOutputBuffering', + 'orchestra.contrib.webapps.options.PHPRegisterGlobals', + 'orchestra.contrib.webapps.options.PHPPostMaxSize', + 'orchestra.contrib.webapps.options.PHPSendmailPath', + 'orchestra.contrib.webapps.options.PHPSessionBugCompatWarn', + 'orchestra.contrib.webapps.options.PHPSessionAutoStart', + 'orchestra.contrib.webapps.options.PHPSafeMode', + 'orchestra.contrib.webapps.options.PHPSuhosinPostMaxVars', + 'orchestra.contrib.webapps.options.PHPSuhosinGetMaxVars', + 'orchestra.contrib.webapps.options.PHPSuhosinRequestMaxVars', + 'orchestra.contrib.webapps.options.PHPSuhosinSessionEncrypt', + 'orchestra.contrib.webapps.options.PHPSuhosinSimulation', + 'orchestra.contrib.webapps.options.PHPSuhosinExecutorIncludeWhitelist', + 'orchestra.contrib.webapps.options.PHPUploadMaxFileSize', + 'orchestra.contrib.webapps.options.PHPUploadTmpDir', + 'orchestra.contrib.webapps.options.PHPZendExtension', + ), + # lazy loading + choices=lambda : ((o.get_class_path(), o.get_class_path()) for o in webapps.options.AppOption.get_plugins(all=True)), + multiple=True, +) + + +WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST = Setting('WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST', + 'mysql.{}'.format(ORCHESTRA_BASE_DOMAIN), + help_text="Uses ORCHESTRA_BASE_DOMAIN by default.", +) + + +WEBAPPS_MOVE_ON_DELETE_PATH = Setting('WEBAPPS_MOVE_ON_DELETE_PATH', + '' +) + + + +WEBAPPS_CMS_CACHE_DIR = Setting('WEBAPPS_CMS_CACHE_DIR', + '/tmp/orchestra_cms_cache', + help_text="Server-side cache directori for CMS tarballs.", +) diff --git a/orchestra/contrib/webapps/signals.py b/orchestra/contrib/webapps/signals.py new file mode 100644 index 00000000..9bffd7d5 --- /dev/null +++ b/orchestra/contrib/webapps/signals.py @@ -0,0 +1,25 @@ +from django.db.models.signals import pre_save, pre_delete +from django.dispatch import receiver + +from .models import WebApp + +# Admin bulk deletion doesn't call model.delete() +# So, signals are used instead of model method overriding + +@receiver(pre_save, sender=WebApp, dispatch_uid='webapps.type.save') +def type_save(sender, *args, **kwargs): + instance = kwargs['instance'] + # Since a webapp might need to cleanup its old config files, the data + # from the OLD VERSION of the webapp is needed. + if instance.pk: + instance._old_self = type(instance).objects.get(id=instance.pk) + instance.type_instance.save() + +@receiver(pre_delete, sender=WebApp, dispatch_uid='webapps.type.delete') +def type_delete(sender, *args, **kwargs): + instance = kwargs['instance'] + instance._old_self = type(instance).objects.get(id=instance.pk) + try: + instance.type_instance.delete() + except KeyError: + pass diff --git a/orchestra/contrib/webapps/tests/__init__.py b/orchestra/contrib/webapps/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/webapps/tests/functional_tests/__init__.py b/orchestra/contrib/webapps/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/webapps/tests/functional_tests/tests.py b/orchestra/contrib/webapps/tests/functional_tests/tests.py new file mode 100644 index 00000000..72536f15 --- /dev/null +++ b/orchestra/contrib/webapps/tests/functional_tests/tests.py @@ -0,0 +1,130 @@ +import ftplib +import os +import unittest +from io import StringIO + +from django.conf import settings as djsettings +from orchestra.contrib.orchestration.models import Route, Server +from orchestra.contrib.systemusers.backends import UNIXUserController +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, save_response_on_error, snapshot_on_error + +from ... import backends + + +TEST_REST_API = int(os.getenv('TEST_REST_API', '0')) + + +class WebAppMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_MASTER_SERVER', 'localhost') + DEPENDENCIES = ( + 'orchestra.contrib.orchestration', + 'orchestra.contrib.systemusers', + 'orchestra.contrib.webapps', + ) + + def setUp(self): + super(WebAppMixin, self).setUp() + self.add_route() + djsettings.DEBUG = True + + def add_route(self): + server, __ = Server.objects.get_or_create(name=self.MASTER_SERVER) + backend = UNIXUserController.get_name() + Route.objects.get_or_create(backend=backend, match=True, host=server) + backend = self.backend.get_name() + match = 'webapp.type == "%s"' % self.type_value + Route.objects.create(backend=backend, match=match, host=server) + + def upload_webapp(self, name): + try: + ftp = ftplib.FTP(self.MASTER_SERVER) + ftp.login(user=self.account.username, passwd=self.account_password) + ftp.cwd('webapps/%s' % name) + index = StringIO() + index.write(self.page[1]) + index.seek(0) + ftp.storbinary('STOR %s' % self.page[0], index) + index.close() + finally: + ftp.close() + + def test_add(self): + name = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(name) + self.addCleanup(self.delete_webapp, name) + self.upload_webapp(name) + + +class StaticWebAppMixin(object): + backend = backends.static.StaticController + type_value = 'static' + token = random_ascii(100) + page = ( + 'index.html', + 'Hello World! %s \n' % token, + 'Hello World! %s \n' % token, + ) + + +class PHPFPMWebAppMixin(StaticWebAppMixin): + backend = backends.php.PHPController + type_value = 'php5.5' + token = random_ascii(100) + page = ( + 'index.php', + '\n' % token, + 'Hello World! %s' % token, + ) + + +@unittest.skipUnless(TEST_REST_API, "REST API tests") +class RESTWebAppMixin(object): + def setUp(self): + super(RESTWebAppMixin, self).setUp() + self.rest_login() + # create main user + self.save_systemuser() + + @save_response_on_error + def save_systemuser(self): + systemuser = self.rest.systemusers.retrieve().get() + systemuser.update(is_active=True) + + @save_response_on_error + def add_webapp(self, name, options=[]): + self.rest.webapps.create(name=name, type=self.type_value, options=options) + + @save_response_on_error + def delete_webapp(self, name): + self.rest.webapps.retrieve(name=name).delete() + + +class AdminWebAppMixin(WebAppMixin): + def setUp(self): + super(AdminWebAppMixin, self).setUp() + self.admin_login() + # create main user + self.save_systemuser() + + @snapshot_on_error + def save_systemuser(self): + url = '' + + @snapshot_on_error + def add(self, name, password, admin_email): + pass + + +class StaticRESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): + pass + + +class PHPFPMRESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): + pass + + +#class AdminWebAppTest(AdminWebAppMixin, BaseLiveServerTestCase): +# pass + + + diff --git a/orchestra/contrib/webapps/types/__init__.py b/orchestra/contrib/webapps/types/__init__.py new file mode 100644 index 00000000..eb1b7d18 --- /dev/null +++ b/orchestra/contrib/webapps/types/__init__.py @@ -0,0 +1,97 @@ +import importlib +import os +from functools import lru_cache + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra import plugins +from orchestra.plugins.forms import PluginDataForm +from orchestra.utils.python import import_class + +from .. import settings +from ..options import AppOption + + +class AppType(plugins.Plugin, metaclass=plugins.PluginMount): + name = None + verbose_name = "" + help_text= "" + form = PluginDataForm + icon = 'orchestra/icons/apps.png' + unique_name = False + option_groups = (AppOption.FILESYSTEM, AppOption.PROCESS, AppOption.PHP) + plugin_field = 'type' + # TODO generic name like 'execution' ? + + @classmethod + @lru_cache() + def get_plugins(cls, all=False): + if all: + for module in os.listdir(os.path.dirname(__file__)): + if module != '__init__.py' and module[-3:] == '.py': + importlib.import_module('.'+module[:-3], __package__) + plugins = super().get_plugins() + else: + plugins = [] + for cls in settings.WEBAPPS_TYPES: + plugins.append(import_class(cls)) + return plugins + + def validate(self): + """ Unique name validation """ + if self.unique_name: + if not self.instance.pk and type(self.instance).objects.filter(name=self.instance.name, type=self.instance.type).exists(): + raise ValidationError({ + 'name': _("A WordPress blog with this name already exists."), + }) + + @classmethod + @lru_cache() + def get_group_options(cls): + """ Get enabled options based on cls.option_groups """ + groups = AppOption.get_option_groups() + options = [] + for group in cls.option_groups: + group_options = groups[group] + if group is None: + options.insert(0, (group, group_options)) + else: + options.append((group, group_options)) + return options + + @classmethod + def get_group_options_choices(cls): + """ Generates grouped choices ready to use in Field.choices """ + # generators can not be @lru_cache + yield (None, '-------') + for group, options in cls.get_group_options(): + if group is None: + for option in options: + yield (option.name, option.verbose_name) + else: + yield (group, [(op.name, op.verbose_name) for op in options]) + + @classmethod + def get_detail_lookups(cls): + """ {'field_name': (('opt1', _("Option 1"),)} """ + return {} + + def get_detail(self): + return '' + + def save(self): + pass + + def delete(self): + pass + + def get_directive_context(self): + return { + 'app_id': self.instance.id, + 'app_name': self.instance.name, + 'user': self.instance.get_username(), + 'user_id': self.instance.account.main_systemuser_id, + 'home': self.instance.account.main_systemuser.get_home(), + 'account_id': self.instance.account_id, + } diff --git a/orchestra/contrib/webapps/types/cms.py b/orchestra/contrib/webapps/types/cms.py new file mode 100644 index 00000000..03524d74 --- /dev/null +++ b/orchestra/contrib/webapps/types/cms.py @@ -0,0 +1,103 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.urls import reverse +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.contrib.databases.models import Database, DatabaseUser +from orchestra.forms.widgets import SpanWidget +from orchestra.utils.python import random_ascii + +from .php import PHPApp, PHPAppForm, PHPAppSerializer + + +class CMSAppForm(PHPAppForm): + db_name = forms.CharField(label=_("Database name"), + help_text=_("Database exclusively used for this webapp.")) + db_user = forms.CharField(label=_("Database user"), + help_text=_("Database user exclusively used for this webapp.")) + password = forms.CharField(label=_("Password"), + help_text=_("Initial database and WordPress admin password.
    " + "Subsequent changes to the admin password will not be reflected.")) + + def __init__(self, *args, **kwargs): + super(CMSAppForm, self).__init__(*args, **kwargs) + if self.instance: + data = self.instance.data + # DB link + db_name = data.get('db_name') + db_id = data.get('db_id') + db_url = reverse('admin:databases_database_change', args=(db_id,)) + db_link = mark_safe('%s' % (db_url, db_name)) + self.fields['db_name'].widget = SpanWidget(original=db_name, display=db_link) + # DB user link + db_user = data.get('db_user') + db_user_id = data.get('db_user_id') + db_user_url = reverse('admin:databases_databaseuser_change', args=(db_user_id,)) + db_user_link = mark_safe('%s' % (db_user_url, db_user)) + self.fields['db_user'].widget = SpanWidget(original=db_user, display=db_user_link) + + +class CMSAppSerializer(PHPAppSerializer): + db_name = serializers.CharField(label=_("Database name"), required=False) + db_user = serializers.CharField(label=_("Database user"), required=False) + password = serializers.CharField(label=_("Password"), required=False) + db_id = serializers.IntegerField(label=_("Database ID"), required=False) + db_user_id = serializers.IntegerField(label=_("Database user ID"), required=False) + + +class CMSApp(PHPApp): + """ Abstract AppType with common CMS functionality """ + serializer = CMSAppSerializer + change_form = CMSAppForm + change_readonly_fields = ('db_name', 'db_user', 'password',) + db_type = Database.MYSQL + abstract = True + db_prefix = 'cms_' + + def get_db_name(self): + db_name = '%s%s_%s' % (self.db_prefix, self.instance.name, self.instance.account) + # Limit for mysql database names + return db_name[:65] + + def get_db_user(self): + db_name = self.get_db_name() + # Limit for mysql user names + return db_name[:16] + + def get_password(self): + return random_ascii(10) + + def validate(self): + super(CMSApp, self).validate() + create = not self.instance.pk + if create: + db = Database(name=self.get_db_name(), account=self.instance.account, type=self.db_type) + user = DatabaseUser(username=self.get_db_user(), password=self.get_password(), + account=self.instance.account, type=self.db_type) + for obj in (db, user): + try: + obj.full_clean() + except ValidationError as e: + raise ValidationError({ + 'name': e.messages, + }) + + def save(self): + db_name = self.get_db_name() + db_user = self.get_db_user() + password = self.get_password() + db, db_created = self.instance.account.databases.get_or_create(name=db_name, type=self.db_type) + if db_created: + user = DatabaseUser(username=db_user, account=self.instance.account, type=self.db_type) + user.set_password(password) + user.save() + db.users.add(user) + self.instance.data.update({ + 'db_name': db_name, + 'db_user': db_user, + 'password': password, + 'db_id': db.id, + 'db_user_id': user.id, + }) diff --git a/orchestra/contrib/webapps/types/misc.py b/orchestra/contrib/webapps/types/misc.py new file mode 100644 index 00000000..1619bbf3 --- /dev/null +++ b/orchestra/contrib/webapps/types/misc.py @@ -0,0 +1,56 @@ +import os + +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from ..options import AppOption + +from . import AppType +from .php import PHPApp, PHPAppForm, PHPAppSerializer + + +class StaticApp(AppType): + name = 'static' + verbose_name = "Static" + help_text = _("This creates a Static application under ~/webapps/<app_name>
    " + "Apache2 will be used to serve static content and execute CGI files.") + icon = 'orchestra/icons/apps/Static.png' + option_groups = (AppOption.FILESYSTEM,) + + def get_directive(self): + return ('static', self.instance.get_path()) + + +class WebalizerApp(AppType): + name = 'webalizer' + verbose_name = "Webalizer" + directive = ('static', '%(app_path)s%(site_name)s') + help_text = _( + "This creates a Webalizer application under ~/webapps/<app_name>-<site_name>
    " + "Statistics will be collected once this app is mounted into one or more Websites.") + icon = 'orchestra/icons/apps/Stats.png' + option_groups = () + + def get_directive(self): + webalizer_path = os.path.join(self.instance.get_path(), '%(site_name)s') + webalizer_path = os.path.normpath(webalizer_path) + return ('static', webalizer_path) + + +class SymbolicLinkForm(PHPAppForm): + path = forms.CharField(label=_("Path"), widget=forms.TextInput(attrs={'size':'100'}), + help_text=_("Path for the origin of the symbolic link.")) + + +class SymbolicLinkSerializer(PHPAppSerializer): + path = serializers.CharField(label=_("Path")) + + +class SymbolicLinkApp(PHPApp): + name = 'symbolic-link' + verbose_name = "Symbolic link" + form = SymbolicLinkForm + serializer = SymbolicLinkSerializer + icon = 'orchestra/icons/apps/SymbolicLink.png' + change_readonly_fields = ('path',) diff --git a/orchestra/contrib/webapps/types/moodle.py b/orchestra/contrib/webapps/types/moodle.py new file mode 100644 index 00000000..aef8bbce --- /dev/null +++ b/orchestra/contrib/webapps/types/moodle.py @@ -0,0 +1,19 @@ +from django.utils.translation import gettext_lazy as _ + +from .cms import CMSApp + + +class MoodleApp(CMSApp): + name = 'moodle-php' + verbose_name = "Moodle" + help_text = _( + "This installs the latest version of Moodle into the webapp directory.
    " + "A database and database user will automatically be created for this webapp.
    " + "This installer creates a user 'admin' with a randomly generated password.
    " + "The password will be visible in the 'password' field after the installer has finished." + ) + icon = 'orchestra/icons/apps/Moodle.png' + db_prefix = 'modl_' + + def get_detail(self): + return self.instance.data.get('php_version', '') diff --git a/orchestra/contrib/webapps/types/php.py b/orchestra/contrib/webapps/types/php.py new file mode 100644 index 00000000..accdb0a8 --- /dev/null +++ b/orchestra/contrib/webapps/types/php.py @@ -0,0 +1,159 @@ +import os +from collections import OrderedDict + +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.plugins.forms import PluginDataForm +from orchestra.utils.functional import cached +from orchestra.utils.python import OrderedSet + +from .. import settings, utils +from ..options import AppOption + +from . import AppType + + +help_message = _("Version of PHP used to execute this webapp.
    " + "Changing the PHP version may result in application malfunction, " + "make sure that everything continue to work as expected.") + + +class PHPAppForm(PluginDataForm): + php_version = forms.ChoiceField(label=_("PHP version"), + choices=settings.WEBAPPS_PHP_VERSIONS, + initial=settings.WEBAPPS_DEFAULT_PHP_VERSION, + help_text=help_message) + + +class PHPAppSerializer(serializers.Serializer): + php_version = serializers.ChoiceField(label=_("PHP version"), + choices=settings.WEBAPPS_PHP_VERSIONS, + default=settings.WEBAPPS_DEFAULT_PHP_VERSION, + help_text=help_message) + + +class PHPApp(AppType): + name = 'php' + verbose_name = "PHP" + help_text = _("This creates a PHP application under ~/webapps/<app_name>
    ") + form = PHPAppForm + serializer = PHPAppSerializer + icon = 'orchestra/icons/apps/PHP.png' + + DEFAULT_PHP_VERSION = settings.WEBAPPS_DEFAULT_PHP_VERSION + PHP_DISABLED_FUNCTIONS = settings.WEBAPPS_PHP_DISABLED_FUNCTIONS + PHP_ERROR_LOG_PATH = settings.WEBAPPS_PHP_ERROR_LOG_PATH + FPM_LISTEN = settings.WEBAPPS_FPM_LISTEN + FCGID_WRAPPER_PATH = settings.WEBAPPS_FCGID_WRAPPER_PATH + + @property + def is_fpm(self): + return self.get_php_version().endswith('-fpm') + + @property + def is_fcgid(self): + return self.get_php_version().endswith('-cgi') + + def get_detail(self): + return self.instance.data.get('php_version', '') + + @classmethod + def get_detail_lookups(cls): + return { + 'php_version': settings.WEBAPPS_PHP_VERSIONS, + } + + @cached + def get_options(self, merge=settings.WEBAPPS_MERGE_PHP_WEBAPPS): + """ adapter to webapp.get_options that performs merging of PHP options """ + kwargs = { + 'webapp_id': self.instance.pk, + } + if merge: + php_version = self.instance.data.get('php_version', self.DEFAULT_PHP_VERSION) + kwargs = { + # webapp__type is not used because wordpress != php != symlink... + 'webapp__account': self.instance.account_id, + 'webapp__data__contains': '"php_version":"%s"' % php_version, + } + return self.instance.get_options(**kwargs) + + def get_php_init_vars(self, merge=settings.WEBAPPS_MERGE_PHP_WEBAPPS): + """ Prepares PHP options for inclusion on php.ini """ + init_vars = OrderedDict() + options = self.get_options(merge=merge) + php_version_number = float(self.get_php_version_number()) + timeout = None + for name, value in options.items(): + if name == 'timeout': + timeout = value + else: + opt = AppOption.get(name) + # Filter non-deprecated PHP options + if opt.group == opt.PHP and (opt.deprecated or 999) > php_version_number: + init_vars[name] = value + # Disable functions + if self.PHP_DISABLED_FUNCTIONS: + enable_functions = init_vars.pop('enable_functions', None) + enable_functions = OrderedSet(enable_functions.split(',') if enable_functions else ()) + disable_functions = init_vars.pop('disable_functions', None) + disable_functions = OrderedSet(disable_functions.split(',') if disable_functions else ()) + if disable_functions or enable_functions or self.is_fpm: + # FPM: Defining 'disable_functions' or 'disable_classes' will not overwrite previously + # defined php.ini values, but will append the new value + for function in self.PHP_DISABLED_FUNCTIONS: + if function not in enable_functions: + disable_functions.add(function) + init_vars['disable_functions'] = ','.join(disable_functions) + # Process timeout + if timeout: + timeout = max(settings.WEBAPPS_PYTHON_DEFAULT_TIMEOUT, int(timeout)) + # Give a little slack here + timeout = str(timeout-2) + init_vars['max_execution_time'] = timeout + # Custom error log + if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars: + context = self.get_directive_context() + error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context) + init_vars['error_log'] = error_log_path + # Auto update max_post_size + if 'upload_max_filesize' in init_vars: + upload_max_filesize = init_vars['upload_max_filesize'] + post_max_size = init_vars.get('post_max_size', '0') + upload_max_filesize_value = eval(upload_max_filesize.replace('M', '*1024')) + post_max_size_value = eval(post_max_size.replace('M', '*1024')) + init_vars['post_max_size'] = post_max_size + if upload_max_filesize_value > post_max_size_value: + init_vars['post_max_size'] = upload_max_filesize + return init_vars + + def get_directive_context(self): + context = super(PHPApp, self).get_directive_context() + context.update({ + 'php_version': self.get_php_version(), + 'php_version_number': self.get_php_version_number(), + 'php_version_int': int(self.get_php_version_number().replace('.', '')), + }) + return context + + def get_directive(self): + context = self.get_directive_context() + if self.is_fpm: + socket = self.FPM_LISTEN % context + return ('fpm', socket, self.instance.get_path()) + elif self.is_fcgid: + wrapper_path = os.path.normpath(self.FCGID_WRAPPER_PATH % context) + return ('fcgid', self.instance.get_path(), wrapper_path) + else: + php_version = self.get_php_version() + raise ValueError("Unknown directive for php version '%s'" % php_version) + + def get_php_version(self): + default_version = self.DEFAULT_PHP_VERSION + return self.instance.data.get('php_version', default_version) + + def get_php_version_number(self): + php_version = self.get_php_version() + return utils.extract_version_number(php_version) diff --git a/orchestra/contrib/webapps/types/python.py b/orchestra/contrib/webapps/types/python.py new file mode 100644 index 00000000..22ff0297 --- /dev/null +++ b/orchestra/contrib/webapps/types/python.py @@ -0,0 +1,64 @@ +import re + +from django import forms +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from orchestra.plugins.forms import PluginDataForm + +from .. import settings +from ..options import AppOption + +from . import AppType + + +help_message = _("Version of Python used to execute this webapp.
    " + "Changing the Python version may result in application malfunction, " + "make sure that everything continue to work as expected.") + + +class PythonAppForm(PluginDataForm): + python_version = forms.ChoiceField(label=_("Python version"), + choices=settings.WEBAPPS_PYTHON_VERSIONS, + initial=settings.WEBAPPS_DEFAULT_PYTHON_VERSION, + help_text=help_message) + + +class PythonAppSerializer(serializers.Serializer): + python_version = serializers.ChoiceField(label=_("Python version"), + choices=settings.WEBAPPS_PYTHON_VERSIONS, + default=settings.WEBAPPS_DEFAULT_PYTHON_VERSION, + help_text=help_message) + + +class PythonApp(AppType): + name = 'python' + verbose_name = "Python" + help_text = _("This creates a Python application under ~/webapps/<app_name>
    ") + form = PythonAppForm + serializer = PythonAppSerializer + option_groups = (AppOption.FILESYSTEM, AppOption.PROCESS) + icon = 'orchestra/icons/apps/Python.png' + + @classmethod + def get_detail_lookups(cls): + return { + 'python_version': settings.WEBAPPS_PYTHON_VERSIONS, + } + + def get_directive(self): + context = self.get_directive_context() + return ('uwsgi', settings.WEBAPPS_UWSGI_SOCKET % context) + + def get_python_version(self): + default_version = self.DEFAULT_PYTHON_VERSION + return self.instance.data.get('python_version', default_version) + + def get_python_version_number(self): + python_version = self.get_python_version() + number = re.findall(r'[0-9]+\.?[0-9]?', python_version) + if not number: + raise ValueError("No version number matches for '%s'" % python_version) + if len(number) > 1: + raise ValueError("Multiple version number matches for '%s'" % python_version) + return number[0] diff --git a/orchestra/contrib/webapps/types/wordpress.py b/orchestra/contrib/webapps/types/wordpress.py new file mode 100644 index 00000000..926f7acc --- /dev/null +++ b/orchestra/contrib/webapps/types/wordpress.py @@ -0,0 +1,19 @@ +from django.utils.translation import gettext_lazy as _ + +from .cms import CMSApp + + +class WordPressApp(CMSApp): + name = 'wordpress-php' + verbose_name = "WordPress" + help_text = _( + "This installs the latest version of WordPress into the webapp directory.
    " + "A database and database user will automatically be created for this webapp.
    " + "This installer creates a user 'admin' with a randomly generated password.
    " + "The password will be visible in the 'password' field after the installer has finished." + ) + icon = 'orchestra/icons/apps/WordPress.png' + db_prefix = 'wp_' + + def get_detail(self): + return self.instance.data.get('php_version', '') diff --git a/orchestra/contrib/webapps/utils.py b/orchestra/contrib/webapps/utils.py new file mode 100644 index 00000000..35388cee --- /dev/null +++ b/orchestra/contrib/webapps/utils.py @@ -0,0 +1,10 @@ +import re + + +def extract_version_number(version): + number = re.findall(r'[0-9]+\.?[0-9]?', version) + if not number: + raise ValueError("No version number matches for '%s'" % version) + if len(number) > 1: + raise ValueError("Multiple version number matches for '%s'" % version) + return number[0] diff --git a/orchestra/contrib/websites/__init__.py b/orchestra/contrib/websites/__init__.py new file mode 100644 index 00000000..93cab2d3 --- /dev/null +++ b/orchestra/contrib/websites/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.websites.apps.WebsitesConfig' diff --git a/orchestra/contrib/websites/admin.py b/orchestra/contrib/websites/admin.py new file mode 100644 index 00000000..581796d6 --- /dev/null +++ b/orchestra/contrib/websites/admin.py @@ -0,0 +1,139 @@ +from django import forms +from django.contrib import admin +from django.urls import resolve +from django.db.models import Q +from django.utils.encoding import force_str +from django.utils.html import format_html +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from orchestra.admin import ExtendedModelAdmin +from orchestra.admin.actions import disable, enable +from orchestra.admin.utils import admin_link, change_url +from orchestra.contrib.accounts.actions import list_accounts +from orchestra.contrib.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter +from orchestra.forms.widgets import DynamicHelpTextSelect +from orchestra.utils.html import get_on_site_link + +from .directives import SiteDirective +from .filters import HasWebAppsListFilter, HasDomainsFilter +from .forms import WebsiteAdminForm, WebsiteDirectiveInlineFormSet +from .models import Content, Website, WebsiteDirective + + +class WebsiteDirectiveInline(admin.TabularInline): + model = WebsiteDirective + formset = WebsiteDirectiveInlineFormSet + extra = 1 + + DIRECTIVES_HELP_TEXT = { + op.name: force_str(op.help_text) for op in SiteDirective.get_plugins() + } + + def formfield_for_dbfield(self, db_field, **kwargs): + if db_field.name == 'value': + kwargs['widget'] = forms.TextInput(attrs={'size':'100'}) + if db_field.name == 'name': + # Help text based on select widget + target = 'this.id.replace("name", "value")' + kwargs['widget'] = DynamicHelpTextSelect(target, self.DIRECTIVES_HELP_TEXT) + return super(WebsiteDirectiveInline, self).formfield_for_dbfield(db_field, **kwargs) + + +class ContentInline(AccountAdminMixin, admin.TabularInline): + model = Content + extra = 1 + fields = ('webapp', 'webapp_link', 'webapp_type', 'path') + readonly_fields = ('webapp_link', 'webapp_type') + filter_by_account_fields = ['webapp'] + + webapp_link = admin_link('webapp', popup=True) + webapp_link.short_description = _("Web App") + + def webapp_type(self, content): + if not content.pk: + return '' + return content.webapp.get_type_display() + webapp_type.short_description = _("Web App type") + + +class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): + list_display = ( + 'name', 'display_domains', 'display_webapps', 'account_link', 'display_active' + ) + list_filter = ( + 'protocol', IsActiveListFilter, HasWebAppsListFilter, HasDomainsFilter + ) + change_readonly_fields = ('name',) + inlines = (ContentInline, WebsiteDirectiveInline) + filter_horizontal = ['domains'] + fieldsets = ( + (None, { + 'classes': ('extrapretty',), + 'fields': ('account_link', 'name', 'protocol', 'target_server', 'domains', 'is_active', 'comments'), + }), + ) + form = WebsiteAdminForm + filter_by_account_fields = ['domains'] + list_prefetch_related = ('domains', 'content_set__webapp') + search_fields = ('name', 'account__username', 'domains__name', 'content__webapp__name') + actions = (disable, enable, list_accounts) + + @mark_safe + def display_domains(self, website): + domains = [] + for domain in website.domains.all(): + url = '%s://%s' % (website.get_protocol(), domain) + domains.append('%s' % (url, url)) + return '
    '.join(domains) + display_domains.short_description = _("domains") + display_domains.admin_order_field = 'domains' + + @mark_safe + def display_webapps(self, website): + webapps = [] + for content in website.content_set.all(): + site_link = get_on_site_link(content.get_absolute_url()) + webapp = content.webapp + detail = _("Edit Webapp") + ' ' + webapp.get_type_display() + try: + detail += ' ' + webapp.type_instance.get_detail() + except KeyError: + pass + url = change_url(webapp) + name = "%s on %s" % (webapp.name, content.path or '/') + webapp_info = format_html('{} {}', url, detail, name, site_link) + webapps.append(webapp_info) + return '
    '.join(webapps) + display_webapps.short_description = _("Web apps") + + def formfield_for_dbfield(self, db_field, **kwargs): + """ + Exclude domains with exhausted ports + has to be done here, on the form doesn't work because of filter_by_account_fields + """ + formfield = super(WebsiteAdmin, self).formfield_for_dbfield(db_field, **kwargs) + if db_field.name == 'domains': + qset = Q( + Q(websites__protocol=Website.HTTPS_ONLY) | + Q(websites__protocol=Website.HTTP_AND_HTTPS) | Q( + Q(websites__protocol=Website.HTTP) & Q(websites__protocol=Website.HTTPS) + ) + ) + object_id = kwargs['request'].resolver_match.kwargs.get('object_id') + if object_id: + qset = Q(qset & ~Q(websites__pk=object_id)) + formfield.queryset = formfield.queryset.exclude(qset) + return formfield + + def _create_formsets(self, request, obj, change): + """ bind contents formset to directive formset for unique location cross-validation """ + formsets, inline_instances = super(WebsiteAdmin, self)._create_formsets(request, obj, change) + if request.method == 'POST': + contents, directives = formsets + directives.content_formset = contents + return formsets, inline_instances + + +admin.site.register(Website, WebsiteAdmin) diff --git a/orchestra/contrib/websites/api.py b/orchestra/contrib/websites/api.py new file mode 100644 index 00000000..bf0d1fe0 --- /dev/null +++ b/orchestra/contrib/websites/api.py @@ -0,0 +1,25 @@ +from rest_framework import viewsets + +from orchestra.api import router, LogApiMixin +from orchestra.contrib.accounts.api import AccountApiMixin + +from . import settings +from .models import Website +from .serializers import WebsiteSerializer + + +class WebsiteViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): + queryset = Website.objects.prefetch_related('domains', 'content_set__webapp', 'directives').all() + serializer_class = WebsiteSerializer + filter_fields = ('name', 'domains__name') + + def options(self, request): + metadata = super(WebsiteViewSet, self).options(request) + names = ['WEBSITES_OPTIONS', 'WEBSITES_PORT_CHOICES'] + metadata.data['settings'] = { + name.lower(): getattr(settings, name, None) for name in names + } + return metadata + + +router.register(r'websites', WebsiteViewSet) diff --git a/orchestra/contrib/websites/apps.py b/orchestra/contrib/websites/apps.py new file mode 100644 index 00000000..4565dbee --- /dev/null +++ b/orchestra/contrib/websites/apps.py @@ -0,0 +1,19 @@ +from django.apps import AppConfig + +from orchestra.core import services +from orchestra.utils.db import database_ready + + +class WebsitesConfig(AppConfig): + name = 'orchestra.contrib.websites' + + def ready(self): + if database_ready(): +# from django.contrib.contenttypes.models import ContentType +# from .models import Content, Website +# qset = Content.content_type.field.get_limit_choices_to() +# for ct in ContentType.objects.filter(qset): +# relation = GenericRelation('websites.Content') +# ct.model_class().add_to_class('content_set', relation) + from .models import Website + services.register(Website, icon='Applications-internet.png') diff --git a/orchestra/contrib/websites/backends/__init__.py b/orchestra/contrib/websites/backends/__init__.py new file mode 100644 index 00000000..6e57f3a5 --- /dev/null +++ b/orchestra/contrib/websites/backends/__init__.py @@ -0,0 +1,5 @@ +import pkgutil + + +for __, module_name, __ in pkgutil.walk_packages(__path__): + exec('from . import %s' % module_name) diff --git a/orchestra/contrib/websites/backends/apache.py b/orchestra/contrib/websites/backends/apache.py new file mode 100644 index 00000000..40aafdee --- /dev/null +++ b/orchestra/contrib/websites/backends/apache.py @@ -0,0 +1,503 @@ +import os +import re +import textwrap + +from django.template import Template, Context +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.resources import ServiceMonitor + +from .. import settings +from ..utils import normurlpath + + +class Apache2Controller(ServiceController): + """ + Apache ≥2.4 backend with support for the following directives: + static, location, fpm, fcgid, uwsgi, \ + ssl, security, redirects, proxies, saas + """ + HTTP_PORT = 80 + HTTPS_PORT = 443 + + model = 'websites.Website' + related_models = ( + ('websites.Content', 'website'), + ('websites.WebsiteDirective', 'website'), + ('webapps.WebApp', 'website_set'), + ) + verbose_name = _("Apache 2") + doc_settings = (settings, ( + 'WEBSITES_VHOST_EXTRA_DIRECTIVES', + 'WEBSITES_DEFAULT_SSL_CERT', + 'WEBSITES_DEFAULT_SSL_KEY', + 'WEBSITES_DEFAULT_SSL_CA', + 'WEBSITES_BASE_APACHE_CONF', + 'WEBSITES_DEFAULT_IPS', + 'WEBSITES_SAAS_DIRECTIVES', + )) + + def get_extra_conf(self, site, context, ssl=False): + extra_conf = self.get_content_directives(site, context) + directives = site.get_directives() + if ssl: + extra_conf += self.get_ssl(directives) + extra_conf += self.get_security(directives) + extra_conf += self.get_redirects(directives) + extra_conf += self.get_proxies(directives) + extra_conf += self.get_saas(directives) + settings_context = site.get_settings_context() + for location, directive in settings.WEBSITES_VHOST_EXTRA_DIRECTIVES: + extra_conf.append((location, directive % settings_context)) + # Order extra conf directives based on directives (longer first) + extra_conf = sorted(extra_conf, key=lambda a: len(a[0]), reverse=True) + return '\n'.join([conf for location, conf in extra_conf]) + + def render_virtual_host(self, site, context, ssl=False): + context.update({ + 'port': self.HTTPS_PORT if ssl else self.HTTP_PORT, + 'vhost_set_fcgid': False, + 'server_alias_lines': ' \\\n '.join(context['server_alias']), + 'suexec_needed': site.target_server == 'web.pangea.lan' + }) + context['extra_conf'] = self.get_extra_conf(site, context, ssl) + return Template(textwrap.dedent("""\ + + IncludeOptional /etc/apache2/site[s]-override/{{ site_unique_name }}.con[f] + ServerName {{ server_name }}\ + {% if server_alias %} + ServerAlias {{ server_alias_lines }}{% endif %}\ + {% if access_log %} + CustomLog {{ access_log }} common{% endif %}\ + {% if error_log %} + ErrorLog {{ error_log }}{% endif %} + {% if suexec_needed %} + SuexecUserGroup {{ user }} {{ group }}{% endif %}\ + {% for line in extra_conf.splitlines %} + {{ line | safe }}{% endfor %} + + """) + ).render(Context(context)) + + def render_redirect_https(self, context): + context['port'] = self.HTTP_PORT + return Template(textwrap.dedent(""" + + ServerName {{ server_name }}\ + {% if server_alias %} + ServerAlias {{ server_alias|join:' ' }}{% endif %}\ + {% if access_log %} + CustomLog {{ access_log }} common{% endif %}\ + {% if error_log %} + ErrorLog {{ error_log }}{% endif %} + RewriteEngine On + RewriteCond %{HTTPS} off + RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} + + """) + ).render(Context(context)) + + def save(self, site): + context = self.get_context(site) + if context['server_name']: + apache_conf = '# %(banner)s\n' % context + if site.protocol in (site.HTTP, site.HTTP_AND_HTTPS): + apache_conf += self.render_virtual_host(site, context, ssl=False) + if site.protocol in (site.HTTP_AND_HTTPS, site.HTTPS_ONLY, site.HTTPS): + apache_conf += self.render_virtual_host(site, context, ssl=True) + if site.protocol == site.HTTPS_ONLY: + apache_conf += self.render_redirect_https(context) + context['apache_conf'] = apache_conf.strip() + self.append(textwrap.dedent(""" + # Generate Apache config for site %(site_name)s + read -r -d '' apache_conf << 'EOF' || true + %(apache_conf)s + EOF + { + echo -e "${apache_conf}" | diff -N -I'^\s*#' %(sites_available)s - + } || { + echo -e "${apache_conf}" > %(sites_available)s + UPDATED_APACHE=1 + }""") % context + ) + if context['server_name'] and site.active: + self.append(textwrap.dedent(""" + # Enable site %(site_name)s + [[ $(a2ensite %(site_unique_name)s) =~ "already enabled" ]] || UPDATED_APACHE=1\ + """) % context + ) + else: + self.append(textwrap.dedent(""" + # Disable site %(site_name)s + [[ $(a2dissite %(site_unique_name)s) =~ "already disabled" ]] || UPDATED_APACHE=1\ + """) % context + ) + + def delete(self, site): + context = self.get_context(site) + self.append(textwrap.dedent(""" + # Remove site configuration for %(site_name)s + [[ $(a2dissite %(site_unique_name)s) =~ "already disabled" ]] || UPDATED_APACHE=1 + rm -f %(sites_available)s\ + """) % context + ) + + def prepare(self): + super(Apache2Controller, self).prepare() + # Coordinate apache restart with php backend in order not to overdo it + self.append(textwrap.dedent(""" + BACKEND="Apache2Controller" + echo "$BACKEND" >> /dev/shm/reload.apache2 + + function coordinate_apache_reload () { + # Coordinate Apache reload with other concurrent backends (e.g. PHPController) + is_last=0 + counter=0 + while ! mv /dev/shm/reload.apache2 /dev/shm/reload.apache2.locked; do + if [[ $counter -gt 4 ]]; then + echo "[ERROR]: Apache reload synchronization deadlocked!" >&2 + exit 10 + fi + counter=$(($counter+1)) + sleep 0.1; + done + state="$(grep -v -E "^$BACKEND($|\s)" /dev/shm/reload.apache2.locked)" || is_last=1 + [[ $is_last -eq 0 ]] && { + echo "$state" | grep -v ' RELOAD$' || is_last=1 + } + if [[ $is_last -eq 1 ]]; then + echo "[DEBUG]: Last backend to run, update: $UPDATED_APACHE, state: '$state'" + if [[ $UPDATED_APACHE -eq 1 || "$state" =~ .*RELOAD$ ]]; then + if service apache2 status > /dev/null; then + service apache2 reload + else + service apache2 start + fi + fi + rm /dev/shm/reload.apache2.locked + else + echo "$state" > /dev/shm/reload.apache2.locked + if [[ $UPDATED_APACHE -eq 1 ]]; then + echo -e "[DEBUG]: Apache will be reloaded by another backend:\\n${state}" + echo "$BACKEND RELOAD" >> /dev/shm/reload.apache2.locked + fi + mv /dev/shm/reload.apache2.locked /dev/shm/reload.apache2 + fi + }""") + ) + + def commit(self): + """ reload Apache2 if necessary """ + self.append("coordinate_apache_reload") + super(Apache2Controller, self).commit() + + def get_directives(self, directive, context): + method, args = directive[0], directive[1:] + try: + method = getattr(self, 'get_%s_directives' % method) + except AttributeError: + context = (self.__class__.__name__, method) + raise AttributeError("%s does not has suport for '%s' directive." % context) + return method(context, *args) + + def get_content_directives(self, site, context): + directives = [] + for content in site.content_set.all(): + directive = content.webapp.get_directive() + self.set_content_context(content, context) + directives += self.get_directives(directive, context) + return directives + + def get_static_directives(self, context, app_path): + context['app_path'] = os.path.normpath(app_path % context) + directive = self.get_location_filesystem_map(context) + return [ + (context['location'], directive), + ] + + def get_location_filesystem_map(self, context): + if not context['location']: + return 'DocumentRoot %(app_path)s' % context + return 'Alias %(location)s %(app_path)s' % context + + def get_fpm_directives(self, context, socket, app_path): + if ':' in socket: + # TCP socket + target = 'fcgi://%(socket)s%(app_path)s/$1' + else: + # UNIX socket + target = 'unix:%(socket)s|fcgi://127.0.0.1/' + context.update({ + 'app_path': os.path.normpath(app_path), + 'socket': socket, + }) + directives = textwrap.dedent(""" + + + SetHandler "proxy:unix:{socket}|fcgi://127.0.0.1" + + + """).format(socket=socket, app_path=app_path) + directives += self.get_location_filesystem_map(context) + return [ + (context['location'], directives), + ] + + def get_fcgid_directives(self, context, app_path, wrapper_path): + context.update({ + 'app_path': os.path.normpath(app_path), + 'wrapper_name': os.path.basename(wrapper_path), + }) + directives = '' + # This Action trick is used instead of FcgidWrapper because we don't want to define + # a new fcgid process class each time an app is mounted (num proc limits enforcement). + if not context['vhost_set_fcgid']: + # fcgi-bin only needs to be defined once per vhots + # We assume that all account wrapper paths will share the same dir + context['wrapper_dir'] = os.path.dirname(wrapper_path) + context['vhost_set_fcgid'] = True + directives = textwrap.dedent("""\ + Alias /fcgi-bin/ %(wrapper_dir)s/ + + SetHandler fcgid-script + Options +ExecCGI + + """) % context + directives += self.get_location_filesystem_map(context) + directives += textwrap.dedent(""" + ProxyPass %(location)s/ ! + + AddHandler php-fcgi .php + Action php-fcgi /fcgi-bin/%(wrapper_name)s + """) % context + return [ + (context['location'], directives), + ] + + def get_uwsgi_directives(self, context, socket): + # requires apache2 mod_proxy_uwsgi + context['socket'] = socket + directives = "ProxyPass / unix:%(socket)s|uwsgi://" % context + directives += self.get_location_filesystem_map(context) + return [ + (context['location'], directives), + ] + + def get_ssl(self, directives): + cert = directives.get('ssl-cert') + key = directives.get('ssl-key') + ca = directives.get('ssl-ca') + if not (cert and key): + cert = [settings.WEBSITES_DEFAULT_SSL_CERT] + key = [settings.WEBSITES_DEFAULT_SSL_KEY] + # Disabled because since the migration to LE, CA is not required here + #ca = [settings.WEBSITES_DEFAULT_SSL_CA] + if not (cert and key): + return [] + ssl_config = [ + "SSLEngine on", + "SSLCertificateFile %s" % cert[0], + "SSLCertificateKeyFile %s" % key[0], + ] + if ca: + ssl_config.append("SSLCACertificateFile %s" % ca[0]) + return [ + ('', '\n'.join(ssl_config)), + ] + + def get_security(self, directives): + rules = [] + location = '/' + for values in directives.get('sec-rule-remove', []): + for rule in values.split(): + rules.append('SecRuleRemoveById %i' % int(rule)) + for location in directives.get('sec-engine', []): + if location == '/': + rules.append('SecRuleEngine Off') + else: + rules.append(textwrap.dedent("""\ + + SecRuleEngine Off + """) % location + ) + security = [] + if rules: + rules = textwrap.dedent("""\ + + %s + """) % '\n '.join(rules) + security.append((location, rules)) + return security + + def get_redirects(self, directives): + redirects = [] + for redirect in directives.get('redirect', []): + location, target = redirect.split() + if re.match(r'^.*[\^\*\$\?\)]+.*$', redirect): + redirect = "RedirectMatch %s %s" % (location, target) + else: + redirect = "Redirect %s %s" % (location, target) + redirects.append( + (location, redirect) + ) + return redirects + + def get_proxies(self, directives): + proxies = [] + for proxy in directives.get('proxy', []): + proxy = proxy.split() + location = proxy[0] + target = proxy[1] + options = ' '.join(proxy[2:]) + location = normurlpath(location) + proxy = textwrap.dedent("""\ + ProxyPass {location}/ {target} {options} + ProxyPassReverse {location}/ {target}""".format( + location=location, target=target, options=options) + ) + proxies.append( + (location, proxy) + ) + return proxies + + def get_saas(self, directives): + saas = [] + for name, values in directives.items(): + if name.endswith('-saas'): + for value in values: + context = { + 'location': normurlpath(value), + } + directive = settings.WEBSITES_SAAS_DIRECTIVES[name] + saas += self.get_directives(directive, context) + return saas + + def get_username(self, site): + option = site.get_directives().get('user_group') + if option: + return option[0] + return site.get_username() + + def get_groupname(self, site): + option = site.get_directives().get('user_group') + if option and ' ' in option: + user, group = option.split() + return group + return site.get_groupname() + + def get_server_names(self, site): + server_name = None + server_alias = [] + for domain in site.domains.all().order_by('name'): + if not server_name and not domain.name.startswith('*'): + server_name = domain.name + else: + server_alias.append(domain.name) + return server_name, server_alias + + def get_context(self, site): + base_apache_conf = settings.WEBSITES_BASE_APACHE_CONF + sites_available = os.path.join(base_apache_conf, 'sites-available') + sites_enabled = os.path.join(base_apache_conf, 'sites-enabled') + server_name, server_alias = self.get_server_names(site) + context = { + 'site': site, + 'site_name': site.name, + 'ips': settings.WEBSITES_DEFAULT_IPS, + 'site_unique_name': site.unique_name, + 'user': self.get_username(site), + 'group': self.get_groupname(site), + 'server_name': server_name, + 'server_alias': server_alias, + 'sites_enabled': "%s.conf" % os.path.join(sites_enabled, site.unique_name), + 'sites_available': "%s.conf" % os.path.join(sites_available, site.unique_name), + 'access_log': site.get_www_access_log_path(), + 'error_log': site.get_www_error_log_path(), + 'banner': self.get_banner(), + } + if not context['ips']: + raise ValueError("WEBSITES_DEFAULT_IPS is empty.") + return context + + def set_content_context(self, content, context): + content_context = { + 'type': content.webapp.type, + 'location': normurlpath(content.path), + 'app_name': content.webapp.name, + 'app_path': content.webapp.get_path(), + } + context.update(content_context) + + +class Apache2Traffic(ServiceMonitor): + """ + Parses apache logs, + looking for the size of each request on the last word of the log line. + """ + model = 'websites.Website' + resource = ServiceMonitor.TRAFFIC + verbose_name = _("Apache 2 Traffic") + monthly_sum_old_values = True + doc_settings = (settings, + ('WEBSITES_TRAFFIC_IGNORE_HOSTS',) + ) + + def prepare(self): + super(Apache2Traffic, self).prepare() + ignore_hosts = '\\|'.join(settings.WEBSITES_TRAFFIC_IGNORE_HOSTS) + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'ignore_hosts': '-v "%s"' % ignore_hosts if ignore_hosts else '', + } + self.append(textwrap.dedent("""\ + function monitor () { + OBJECT_ID=$1 + INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") + END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s') + LOG_FILE="$3" + { + { grep %(ignore_hosts)s ${LOG_FILE} || echo -e '\\r'; } \\ + | awk -v ini="${INI_DATE}" -v end="${END_DATE}" ' + BEGIN { + sum = 0 + months["Jan"] = "01" + months["Feb"] = "02" + months["Mar"] = "03" + months["Apr"] = "04" + months["May"] = "05" + months["Jun"] = "06" + months["Jul"] = "07" + months["Aug"] = "08" + months["Sep"] = "09" + months["Oct"] = "10" + months["Nov"] = "11" + months["Dec"] = "12" + } { + # date = [11/Jul/2014:13:50:41 + date = substr($4, 2) + year = substr(date, 8, 4) + month = months[substr(date, 4, 3)]; + day = substr(date, 1, 2) + hour = substr(date, 13, 2) + minute = substr(date, 16, 2) + second = substr(date, 19, 2) + line_date = year month day hour minute second + if ( line_date > ini && line_date < end) + sum += $NF + } END { + print sum + }' || [[ $? == 1 ]] && true + } | xargs echo ${OBJECT_ID} + }""") % context) + + def monitor(self, site): + context = self.get_context(site) + self.append('monitor {object_id} "{last_date}" {log_file}'.format(**context)) + + def get_context(self, site): + return { + 'log_file': '%s{,.1}' % site.get_www_access_log_path(), + 'last_date': self.get_last_date(site.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'object_id': site.pk, + } diff --git a/orchestra/contrib/websites/backends/moodle.py b/orchestra/contrib/websites/backends/moodle.py new file mode 100644 index 00000000..b44ec5e0 --- /dev/null +++ b/orchestra/contrib/websites/backends/moodle.py @@ -0,0 +1,25 @@ +import textwrap + +from orchestra.contrib.orchestration import ServiceController + + +class MoodleWWWRootController(ServiceController): + """ + Configures Moodle site WWWRoot, without it Moodle refuses to work. + """ + verbose_name = "Moodle WWWRoot (required)" + model = 'websites.Content' + default_route_match = "content.webapp.type == 'moodle-php'" + + def save(self, content): + context = self.get_context(content) + self.append(textwrap.dedent("""\ + sed -i "s#wwwroot\s*= '.*#wwwroot = '%(url)s';#" %(app_path)s/config.php + """) % context + ) + + def get_context(self, content): + return { + 'url': content.get_absolute_url(), + 'app_path': content.webapp.get_path(), + } diff --git a/orchestra/contrib/websites/backends/webalizer.py b/orchestra/contrib/websites/backends/webalizer.py new file mode 100644 index 00000000..093aa6bd --- /dev/null +++ b/orchestra/contrib/websites/backends/webalizer.py @@ -0,0 +1,103 @@ +import os +import textwrap + +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.orchestration import ServiceController + +from .. import settings + + +class WebalizerController(ServiceController): + """ + Creates webalizer conf file for each time a webalizer webapp is mounted on a website. + """ + verbose_name = _("Webalizer Content") + model = 'websites.Content' + default_route_match = "content.webapp.type == 'webalizer'" + doc_settings = (settings, + ('WEBSITES_WEBALIZER_PATH',) + ) + + def save(self, content): + context = self.get_context(content) + self.append(textwrap.dedent("""\ + mkdir -p %(webalizer_path)s + if [[ ! -e %(webalizer_path)s/index.html ]]; then + echo 'Webstats are coming soon' > %(webalizer_path)s/index.html + fi + cat << 'EOF' > %(webalizer_conf_path)s + %(webalizer_conf)s + EOF + chown %(user)s:www-data %(webalizer_path)s + chmod g+xr %(webalizer_path)s + """) % context + ) + + def delete(self, content): + context = self.get_context(content) + delete_webapp = not type(content.webapp).objects.filter(pk=content.webapp.pk).exists() + if delete_webapp: + self.append("rm -fr %(webapp_path)s" % context) + remounted = content.webapp.content_set.filter(website=content.website).exists() + if delete_webapp or not remounted: + self.append("rm -fr %(webalizer_path)s" % context) + self.append("rm -f %(webalizer_conf_path)s" % context) + + def get_context(self, content): + conf_file = "%s.conf" % content.website.unique_name + context = { + 'site_logs': content.website.get_www_access_log_path(), + 'site_name': content.website.name, + 'webapp_path': content.webapp.get_path(), + 'webalizer_path': os.path.join(content.webapp.get_path(), content.website.name), + 'webalizer_conf_path': os.path.join(settings.WEBSITES_WEBALIZER_PATH, conf_file), + 'user': content.webapp.account.username, + 'banner': self.get_banner(), + } + context['webalizer_conf'] = textwrap.dedent("""\ + # %(banner)s + LogFile %(site_logs)s + LogType clf + OutputDir %(webalizer_path)s + HistoryName webalizer.hist + Incremental yes + IncrementalName webalizer.current + ReportTitle Stats of + HostName %(site_name)s + + PageType htm* + PageType php* + PageType shtml + PageType cgi + PageType pl + + DNSCache /var/lib/dns_cache.db + DNSChildren 15 + + HideURL *.gif + HideURL *.GIF + HideURL *.jpg + HideURL *.JPG + HideURL *.png + HideURL *.PNG + HideURL *.ra + + IncludeURL * + + SearchEngine yahoo.com p= + SearchEngine altavista.com q= + SearchEngine google.com q= + SearchEngine eureka.com q= + SearchEngine lycos.com query= + SearchEngine hotbot.com MT= + SearchEngine msn.com MT= + SearchEngine infoseek.com qt= + SearchEngine webcrawler searchText= + SearchEngine excite search= + SearchEngine netscape.com search= + SearchEngine mamma.com query= + SearchEngine alltheweb.com query= + + DumpSites yes""") % context + return context diff --git a/orchestra/contrib/websites/backends/wordpress.py b/orchestra/contrib/websites/backends/wordpress.py new file mode 100644 index 00000000..d138e9b7 --- /dev/null +++ b/orchestra/contrib/websites/backends/wordpress.py @@ -0,0 +1,66 @@ +import os +import textwrap + +from orchestra.contrib.orchestration import ServiceController + + +class WordPressURLController(ServiceController): + """ + Configures WordPress site URL with associated website domain. + """ + verbose_name = "WordPress URL" + model = 'websites.Content' + default_route_match = "content.webapp.type == 'wordpress-php'" + + def save(self, content): + context = self.get_context(content) + if context['url']: + self.append(textwrap.dedent("""\ + mysql %(db_name)s -e 'UPDATE wp_options + SET option_value="%(url)s" + WHERE option_id IN (1, 2) AND option_value="http:";' + """) % context + ) + + def delete(self, content): + context = self.get_context(content) + self.append(textwrap.dedent("""\ + mysql %(db_name)s -e 'UPDATE wp_options + SET option_value="http:" + WHERE option_id IN (1, 2);' + """) % context + ) + + def get_context(self, content): + return { + 'url': content.get_absolute_url(), + 'db_name': content.webapp.data.get('db_name'), + } + + +class WordPressForceSSLController(ServiceController): + """ sets FORCE_SSL_ADMIN to true when website supports HTTPS """ + verbose_name = "WordPress Force SSL" + model = 'websites.Content' + related_models = ( + ('websites.Website', 'content_set'), + ) + default_route_match = "content.webapp.type == 'wordpress-php'" + + def save(self, content): + context = self.get_context(content) + site = content.website + if site.protocol in (site.HTTP_AND_HTTPS, site.HTTPS_ONLY, site.HTTPS): + self.append(textwrap.dedent(""" + if [[ ! $(grep FORCE_SSL_ADMIN %(wp_conf_path)s) ]]; then + echo "Enabling FORCE_SSL_ADMIN for %(webapp_name)s webapp" + sed -i -E "s#^(define\('NONCE_SALT.*)#\\1\\n\\ndefine\('FORCE_SSL_ADMIN', true\);#" \\ + %(wp_conf_path)s + fi""") % context + ) + + def get_context(self, content): + return { + 'webapp_name': content.webapp.name, + 'wp_conf_path': os.path.join(content.webapp.get_path(), 'wp-config.php'), + } diff --git a/orchestra/contrib/websites/directives.py b/orchestra/contrib/websites/directives.py new file mode 100644 index 00000000..6192d164 --- /dev/null +++ b/orchestra/contrib/websites/directives.py @@ -0,0 +1,207 @@ +import re +from collections import defaultdict +from functools import lru_cache + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from orchestra import plugins +from orchestra.utils.python import import_class + +from . import settings +from .utils import normurlpath + + +class SiteDirective(plugins.Plugin, metaclass=plugins.PluginMount): + HTTPD = 'HTTPD' + SEC = 'ModSecurity' + SSL = 'SSL' + SAAS = 'SaaS' + + help_text = "" + unique_name = False + unique_value = False + is_location = False + + @classmethod + @lru_cache() + def get_plugins(cls, all=False): + if all: + plugins = super().get_plugins() + else: + plugins = [] + for cls in settings.WEBSITES_ENABLED_DIRECTIVES: + plugins.append(import_class(cls)) + return plugins + + @classmethod + @lru_cache() + def get_option_groups(cls): + groups = {} + for opt in cls.get_plugins(): + try: + groups[opt.group].append(opt) + except KeyError: + groups[opt.group] = [opt] + return groups + + @classmethod + def get_choices(cls): + """ Generates grouped choices ready to use in Field.choices """ + # generators can not be @lru_cache() + yield (None, '-------') + options = cls.get_option_groups() + for option in options.pop(None, ()): + yield (option.name, option.verbose_name) + for group, options in options.items(): + yield (group, [(op.name, op.verbose_name) for op in options]) + + def validate_uniqueness(self, directive, values, locations): + """ Validates uniqueness location, name and value """ + errors = defaultdict(list) + value = directive.get('value', None) + # location uniqueness + location = None + if self.is_location and value is not None: + if not value and self.is_location: + value = '/' + location = normurlpath(value.split()[0]) + if location is not None and location in locations: + errors['value'].append(ValidationError( + "Location '%s' already in use by other content/directive." % location + )) + else: + locations.add(location) + + # name uniqueness + if self.unique_name and self.name in values: + errors[None].append(ValidationError( + _("Only one %s can be defined.") % self.get_verbose_name() + )) + + # value uniqueness + if value is not None: + if self.unique_value and value in values.get(self.name, []): + errors['value'].append(ValidationError( + _("This value is already used by other %s.") % force_str(self.get_verbose_name()) + )) + values[self.name].append(value) + if errors: + raise ValidationError(errors) + + def validate(self, directive): + directive.value = directive.value.strip() + if not directive.value and self.is_location: + directive.value = '/' + if self.regex and not re.match(self.regex, directive.value): + raise ValidationError({ + 'value': ValidationError(_("'%(value)s' does not match %(regex)s."), + params={ + 'value': directive.value, + 'regex': self.regex + }), + }) + + +class Redirect(SiteDirective): + name = 'redirect' + verbose_name = _("Redirection") + help_text = _("<website path> <destination URL>") + regex = r'^[^ ]*\s[^ ]+$' + group = SiteDirective.HTTPD + unique_value = True + is_location = True + + def validate(self, directive): + """ inserts default url-path if not provided """ + values = directive.value.strip().split() + if len(values) == 1: + values.insert(0, '/') + directive.value = ' '.join(values) + super(Redirect, self).validate(directive) + + +class Proxy(Redirect): + name = 'proxy' + verbose_name = _("Proxy") + help_text = _("<website path> <target URL>") + regex = r'^[^ ]+\shttp[^ ]+(timeout=[0-9]{1,3}|retry=[0-9]|\s)*$' + + +class ErrorDocument(SiteDirective): + name = 'error-document' + verbose_name = _("ErrorDocumentRoot") + help_text = _("<error code> <URL/path/message>
    " + " 500 http://foo.example.com/cgi-bin/tester
    " + " 404 /cgi-bin/bad_urls.pl
    " + " 401 /subscription_info.html
    " + " 403 \"Sorry can't allow you access today\"") + regex = r'[45]0[0-9]\s.*' + group = SiteDirective.HTTPD + unique_value = True + + +class SSLCA(SiteDirective): + name = 'ssl-ca' + verbose_name = _("SSL CA") + help_text = _("Filesystem path of the CA certificate file.") + regex = r'^/[^ ]+$' + group = SiteDirective.SSL + unique_name = True + + +class SSLCert(SSLCA): + name = 'ssl-cert' + verbose_name = _("SSL cert") + help_text = _("Filesystem path of the certificate file.") + + +class SSLKey(SSLCA): + name = 'ssl-key' + verbose_name = _("SSL key") + help_text = _("Filesystem path of the key file.") + + +class SecRuleRemove(SiteDirective): + name = 'sec-rule-remove' + verbose_name = _("SecRuleRemoveById") + help_text = _("Space separated ModSecurity rule IDs.") + regex = r'^[0-9\s]+$' + group = SiteDirective.SEC + is_location = True + + +class SecEngine(SecRuleRemove): + name = 'sec-engine' + verbose_name = _("SecRuleEngine Off") + help_text = _("URL-path with disabled modsecurity engine.") + regex = r'^/[^ ]*$' + is_location = False + + +class WordPressSaaS(SiteDirective): + name = 'wordpress-saas' + verbose_name = "WordPress SaaS" + help_text = _("URL-path for mounting WordPress multisite.") + group = SiteDirective.SAAS + regex = r'^/[^ ]*$' + unique_value = True + is_location = True + + +class DokuWikiSaaS(WordPressSaaS): + name = 'dokuwiki-saas' + verbose_name = "DokuWiki SaaS" + help_text = _("URL-path for mounting DokuWiki multisite.") + + +class DrupalSaaS(WordPressSaaS): + name = 'drupal-saas' + verbose_name = "Drupdal SaaS" + help_text = _("URL-path for mounting Drupal multisite.") + + +class MoodleSaaS(WordPressSaaS): + name = 'moodle-saas' + verbose_name = "Moodle SaaS" + help_text = _("URL-path for mounting Moodle multisite.") diff --git a/orchestra/contrib/websites/filters.py b/orchestra/contrib/websites/filters.py new file mode 100644 index 00000000..11016195 --- /dev/null +++ b/orchestra/contrib/websites/filters.py @@ -0,0 +1,34 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import gettext_lazy as _ + + +class HasWebAppsListFilter(SimpleListFilter): + """ Filter addresses whether they have any webapp or not """ + title = _("has webapps") + parameter_name = 'has_webapps' + + def lookups(self, request, model_admin): + return ( + ('True', _("True")), + ('False', _("False")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(content__isnull=False) + elif self.value() == 'False': + return queryset.filter(content__isnull=True) + return queryset + + +class HasDomainsFilter(HasWebAppsListFilter): + """ Filter addresses whether they have any domains or not """ + title = _("has domains") + parameter_name = 'has_domains' + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(domains__isnull=False) + elif self.value() == 'False': + return queryset.filter(domains__isnull=True) + return queryset diff --git a/orchestra/contrib/websites/forms.py b/orchestra/contrib/websites/forms.py new file mode 100644 index 00000000..fc4a5029 --- /dev/null +++ b/orchestra/contrib/websites/forms.py @@ -0,0 +1,50 @@ +from collections import defaultdict + +from django import forms +from django.core.exceptions import ValidationError + +from .utils import normurlpath +from .validators import validate_domain_protocol, validate_server_name + + +class WebsiteAdminForm(forms.ModelForm): + def clean(self): + """ Prevent multiples domains on the same protocol """ + super(WebsiteAdminForm, self).clean() + domains = self.cleaned_data.get('domains') + if not domains: + return self.cleaned_data + protocol = self.cleaned_data.get('protocol') + domains = domains.all() + for domain in domains: + try: + validate_domain_protocol(self.instance, domain, protocol) + except ValidationError as err: + self.add_error(None, err) + try: + validate_server_name(domains) + except ValidationError as err: + self.add_error('domains', err) + return self.cleaned_data + + +class WebsiteDirectiveInlineFormSet(forms.models.BaseInlineFormSet): + def clean(self): + # directives formset cross-validation with contents for unique locations + locations = set() + for form in self.content_formset.forms: + location = form.cleaned_data.get('path') + delete = form.cleaned_data.get('DELETE') + if not delete and location is not None: + locations.add(normurlpath(location)) + + values = defaultdict(list) + for form in self.forms: + wdirective = form.instance + directive = form.cleaned_data + if directive.get('name') is not None: + try: + wdirective.directive_instance.validate_uniqueness(directive, values, locations) + except ValidationError as err: + for k,v in err.error_dict.items(): + form.add_error(k, v) diff --git a/orchestra/contrib/websites/models.py b/orchestra/contrib/websites/models.py new file mode 100644 index 00000000..b9f00bbd --- /dev/null +++ b/orchestra/contrib/websites/models.py @@ -0,0 +1,177 @@ +import os +from collections import OrderedDict + +from django.db import models +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ + +from orchestra.core import validators +from orchestra.utils.functional import cached + +from . import settings +from .directives import SiteDirective + + +class Website(models.Model): + """ Models a web site, also known as virtual host """ + HTTP = 'http' + HTTPS = 'https' + HTTP_AND_HTTPS = 'http/https' + HTTPS_ONLY = 'https-only' + + name = models.CharField(_("name"), max_length=128, + validators=[validators.validate_name]) + account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, + verbose_name=_("Account"), related_name='websites') + protocol = models.CharField(_("protocol"), max_length=16, + choices=settings.WEBSITES_PROTOCOL_CHOICES, + default=settings.WEBSITES_DEFAULT_PROTOCOL, + help_text=_("Select the protocol(s) for this website
    " + "HTTPS only performs a redirection from http to https.")) +# port = models.PositiveIntegerField(_("port"), +# choices=settings.WEBSITES_PORT_CHOICES, +# default=settings.WEBSITES_DEFAULT_PORT) + domains = models.ManyToManyField(settings.WEBSITES_DOMAIN_MODEL, blank=True, + related_name='websites', verbose_name=_("domains")) + contents = models.ManyToManyField('webapps.WebApp', through='websites.Content') + target_server = models.ForeignKey('orchestration.Server', on_delete=models.CASCADE, + verbose_name=_("Target Server"), related_name='websites') + is_active = models.BooleanField(_("active"), default=True) + comments = models.TextField(default="", blank=True) + + class Meta: + unique_together = ('name', 'account') + + def __str__(self): + return self.name + + @property + def unique_name(self): + context = self.get_settings_context() + return settings.WEBSITES_UNIQUE_NAME_FORMAT % context + + @cached_property + def active(self): + return self.is_active and self.account.is_active + + def disable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def enable(self): + self.is_active = False + self.save(update_fields=('is_active',)) + + def get_settings_context(self): + """ format settings strings """ + return { + 'id': self.id, + 'pk': self.pk, + 'home': self.get_user().get_home(), + 'user': self.get_username(), + 'group': self.get_groupname(), + 'site_name': self.name, + 'protocol': self.protocol, + } + + def get_protocol(self): + if self.protocol in (self.HTTP, self.HTTP_AND_HTTPS): + return self.HTTP + return self.HTTPS + + @cached + def get_directives(self): + directives = OrderedDict() + for opt in self.directives.all().order_by('name', 'value'): + try: + directives[opt.name].append(opt.value) + except KeyError: + directives[opt.name] = [opt.value] + return directives + + def get_absolute_url(self): + try: + domain = self.domains.all()[0] + except IndexError: + return + else: + return '%s://%s' % (self.get_protocol(), domain) + + def get_user(self): + return self.account.main_systemuser + + def get_username(self): + return self.get_user().username + + def get_groupname(self): + return self.get_username() + + def get_www_access_log_path(self): + context = self.get_settings_context() + context['unique_name'] = self.unique_name + path = settings.WEBSITES_WEBSITE_WWW_ACCESS_LOG_PATH % context + return os.path.normpath(path) + + def get_www_error_log_path(self): + context = self.get_settings_context() + context['unique_name'] = self.unique_name + path = settings.WEBSITES_WEBSITE_WWW_ERROR_LOG_PATH % context + return os.path.normpath(path) + + +class WebsiteDirective(models.Model): + website = models.ForeignKey(Website, on_delete=models.CASCADE, + verbose_name=_("web site"), related_name='directives') + name = models.CharField(_("name"), max_length=128, db_index=True, + choices=SiteDirective.get_choices()) + value = models.CharField(_("value"), max_length=256, blank=True) + + def __str__(self): + return self.name + + @cached_property + def directive_class(self): + return SiteDirective.get(self.name) + + @cached_property + def directive_instance(self): + """ Per request lived directive instance """ + return self.directive_class() + + def clean(self): + self.directive_instance.validate(self) + + +class Content(models.Model): + # related_name is content_set to differentiate between website.content -> webapp + webapp = models.ForeignKey('webapps.WebApp', on_delete=models.CASCADE, + verbose_name=_("web application")) + website = models.ForeignKey('websites.Website', on_delete=models.CASCADE, + verbose_name=_("web site")) + path = models.CharField(_("path"), max_length=256, blank=True, + validators=[validators.validate_url_path]) + + class Meta: + unique_together = ('website', 'path') + + def __str__(self): + try: + return self.website.name + self.path + except Website.DoesNotExist: + return self.path + + def clean_fields(self, *args, **kwargs): + self.path = self.path.strip() + return super(Content, self).clean_fields(*args, **kwargs) + + def clean(self): + if not self.path: + self.path = '/' + + def get_absolute_url(self): + try: + domain = self.website.domains.all()[0] + except IndexError: + return + else: + return '%s://%s%s' % (self.website.get_protocol(), domain, self.path) diff --git a/orchestra/contrib/websites/serializers.py b/orchestra/contrib/websites/serializers.py new file mode 100644 index 00000000..49eb2b6f --- /dev/null +++ b/orchestra/contrib/websites/serializers.py @@ -0,0 +1,130 @@ +from django.core.exceptions import ValidationError +from rest_framework import serializers + +from orchestra.api.serializers import HyperlinkedModelSerializer, RelatedHyperlinkedModelSerializer +from orchestra.contrib.accounts.serializers import AccountSerializerMixin + +from .directives import SiteDirective +from .models import Website, Content, WebsiteDirective +from .utils import normurlpath +from .validators import validate_domain_protocol + + + +class RelatedDomainSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = Website.domains.field.related_model + fields = ('url', 'id', 'name') + + +class RelatedWebAppSerializer(AccountSerializerMixin, RelatedHyperlinkedModelSerializer): + class Meta: + model = Content.webapp.field.related_model + fields = ('url', 'id', 'name', 'type') + + +class ContentSerializer(serializers.ModelSerializer): + webapp = RelatedWebAppSerializer() + + class Meta: + model = Content + fields = ('webapp', 'path') + + def get_identity(self, data): + return '%s-%s' % (data.get('website'), data.get('path')) + + +class DirectiveSerializer(serializers.ModelSerializer): + class Meta: + model = WebsiteDirective + fields = ('name', 'value') + + def to_representation(self, instance): + return {prop.name: prop.value for prop in instance.all()} + + def to_internal_value(self, data): + return data + + +class WebsiteSerializer(AccountSerializerMixin, HyperlinkedModelSerializer): + domains = RelatedDomainSerializer(many=True, required=False) + contents = ContentSerializer(required=False, many=True, source='content_set') + directives = DirectiveSerializer(required=False) + + class Meta: + model = Website + fields = ('url', 'id', 'name', 'protocol', 'domains', 'is_active', 'contents', 'directives') + postonly_fields = ('name',) + + def validate(self, data): + """ Prevent multiples domains on the same protocol """ + # Validate location and directive uniqueness + errors = [] + directives = data.get('directives', []) + if directives: + locations = set() + for content in data.get('content_set', []): + location = content.get('path') + if location is not None: + locations.add(normurlpath(location)) + values = defaultdict(list) + for name, value in directives.items(): + directive = { + 'name': name, + 'value': value, + } + try: + SiteDirective.get(name).validate_uniqueness(directive, values, locations) + except ValidationError as err: + errors.append(err) + # Validate domain protocol uniqueness + instance = self.instance + for domain in data['domains']: + try: + validate_domain_protocol(instance, domain, data['protocol']) + except ValidationError as err: + errors.append(err) + if errors: + raise ValidationError(errors) + return data + + def create(self, validated_data): + directives_data = validated_data.pop('directives') + webapp = super(WebsiteSerializer, self).create(validated_data) + for key, value in directives_data.items(): + WebsiteDirective.objects.create(webapp=webapp, name=key, value=value) + return webap + + def update_directives(self, instance, directives_data): + existing = {} + for obj in instance.directives.all(): + existing[obj.name] = obj + posted = set() + for key, value in directives_data.items(): + posted.add(key) + try: + directive = existing[key] + except KeyError: + directive = instance.directives.create(name=key, value=value) + else: + if directive.value != value: + directive.value = value + directive.save(update_fields=('value',)) + for to_delete in set(existing.keys())-posted: + existing[to_delete].delete() + + def update_contents(self, instance, contents_data): + raise NotImplementedError + + def update_domains(self, instance, domains_data): + raise NotImplementedError + + def update(self, instance, validated_data): + directives_data = validated_data.pop('directives') + domains_data = validated_data.pop('domains') + contents_data = validated_data.pop('content_set') + instance = super(WebsiteSerializer, self).update(instance, validated_data) + self.update_directives(instance, directives_data) + self.update_contents(instance, contents_data) + self.update_domains(instance, domains_data) + return instance diff --git a/orchestra/contrib/websites/settings.py b/orchestra/contrib/websites/settings.py new file mode 100644 index 00000000..147828f1 --- /dev/null +++ b/orchestra/contrib/websites/settings.py @@ -0,0 +1,129 @@ +from django.utils.translation import gettext_lazy as _ + +from orchestra.contrib.settings import Setting +from orchestra.core.validators import validate_ip_address + +from .. import websites + + +_names = ('id', 'pk', 'home', 'user', 'group', 'site_name', 'protocol') +_log_names = _names + ('unique_name',) + + +WEBSITES_UNIQUE_NAME_FORMAT = Setting('WEBSITES_UNIQUE_NAME_FORMAT', + default='%(user)s-%(site_name)s', + help_text="Available fromat names: %s" % ', '.join(_names), + validators=[Setting.string_format_validator(_names)], +) + + +WEBSITES_PROTOCOL_CHOICES = Setting('WEBSITES_PROTOCOL_CHOICES', + default=( + ('http', "HTTP"), + ('https', "HTTPS"), + ('http/https', _("HTTP and HTTPS")), + ('https-only', _("HTTPS only")), + ), + validators=[Setting.validate_choices] +) + + +WEBSITES_DEFAULT_PROTOCOL = Setting('WEBSITES_DEFAULT_PROTOCOL', + default='http', + choices=WEBSITES_PROTOCOL_CHOICES +) + + +WEBSITES_DEFAULT_IPS = Setting('WEBSITES_DEFAULT_IPS', + default=('*',) +) + + +WEBSITES_DOMAIN_MODEL = Setting('WEBSITES_DOMAIN_MODEL', + 'domains.Domain', + validators=[Setting.validate_model_label] +) + + +WEBSITES_ENABLED_DIRECTIVES = Setting('WEBSITES_ENABLED_DIRECTIVES', + ( + 'orchestra.contrib.websites.directives.Redirect', + 'orchestra.contrib.websites.directives.Proxy', + 'orchestra.contrib.websites.directives.ErrorDocument', + 'orchestra.contrib.websites.directives.SSLCA', + 'orchestra.contrib.websites.directives.SSLCert', + 'orchestra.contrib.websites.directives.SSLKey', + 'orchestra.contrib.websites.directives.SecRuleRemove', + 'orchestra.contrib.websites.directives.SecEngine', + 'orchestra.contrib.websites.directives.WordPressSaaS', + 'orchestra.contrib.websites.directives.DokuWikiSaaS', + 'orchestra.contrib.websites.directives.DrupalSaaS', + 'orchestra.contrib.websites.directives.MoodleSaaS', + ), + # lazy loading + choices=lambda : ((d.get_class_path(), d.get_class_path()) for d in websites.directives.SiteDirective.get_plugins(all=True)), + multiple=True, +) + + +WEBSITES_BASE_APACHE_CONF = Setting('WEBSITES_BASE_APACHE_CONF', + '/etc/apache2/' +) + + +WEBSITES_WEBALIZER_PATH = Setting('WEBSITES_WEBALIZER_PATH', + '/home/httpd/webalizer/' +) + + +WEBSITES_WEBSITE_WWW_ACCESS_LOG_PATH = Setting('WEBSITES_WEBSITE_WWW_ACCESS_LOG_PATH', + '/var/log/apache2/virtual/%(unique_name)s.log', + help_text="Available fromat names: %s" % ', '.join(_log_names), + validators=[Setting.string_format_validator(_log_names)], +) + + +WEBSITES_WEBSITE_WWW_ERROR_LOG_PATH = Setting('WEBSITES_WEBSITE_WWW_ERROR_LOG_PATH', + '', + help_text="Available fromat names: %s" % ', '.join(_log_names), + validators=[Setting.string_format_validator(_log_names)], +) + + +WEBSITES_TRAFFIC_IGNORE_HOSTS = Setting('WEBSITES_TRAFFIC_IGNORE_HOSTS', + ('127.0.0.1',), + help_text=_("IP addresses to ignore during traffic accountability."), + validators=[lambda hosts: (validate_ip_address(host) for host in hosts)], +) + + +# TODO sane defaults +WEBSITES_SAAS_DIRECTIVES = Setting('WEBSITES_SAAS_DIRECTIVES', + { + 'wordpress-saas': ('fpm', '/var/run/fpm/pangea-5.4-fpm.sock', '/home/httpd/wordpress-mu/'), + 'drupal-saas': ('fpm', '/var/run/fpm/pangea-5.4-fpm.sock', '/home/httpd/drupal-mu/'), + 'dokuwiki-saas': ('fpm', '/var/run/fpm/pangea-5.4-fpm.sock', '/home/httpd/dokuwiki-mu/'), + 'moodle-saas': ('fpm', '/var/run/fpm/pangea-5.4-fpm.sock', '/home/httpd/moodle-mu/'), + }, +) + + +WEBSITES_DEFAULT_SSL_CERT = Setting('WEBSITES_DEFAULT_SSL_CERT', + '' +) + +WEBSITES_DEFAULT_SSL_KEY = Setting('WEBSITES_DEFAULT_SSL_KEY', + '' +) + +WEBSITES_DEFAULT_SSL_CA = Setting('WEBSITES_DEFAULT_SSL_CA', + '' +) + +WEBSITES_VHOST_EXTRA_DIRECTIVES = Setting('WEBSITES_VHOST_EXTRA_DIRECTIVES', + (), + help_text=( + "(, ),
    " + "i.e. ('/cgi-bin/', 'ScriptAlias /cgi-bin/ %(home)s/cgi-bin/')" + ) +) diff --git a/orchestra/contrib/websites/tests/__init__.py b/orchestra/contrib/websites/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/websites/tests/functional_tests/__init__.py b/orchestra/contrib/websites/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/contrib/websites/tests/functional_tests/tests.py b/orchestra/contrib/websites/tests/functional_tests/tests.py new file mode 100644 index 00000000..1a9bd3ad --- /dev/null +++ b/orchestra/contrib/websites/tests/functional_tests/tests.py @@ -0,0 +1,140 @@ +import os +import socket + +import requests + +from orchestra.contrib.domains.models import Domain, Record +from orchestra.contrib.domains.backends import Bind9MasterDomainController +from orchestra.contrib.orchestration.models import Server, Route +from orchestra.contrib.webapps.tests.functional_tests.tests import StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, PHPFPMWebAppMixin +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, save_response_on_error + +from ... import backends + + +class WebsiteMixin(WebAppMixin): + MASTER_SERVER = os.environ.get('ORCHESTRA_MASTER_SERVER', 'localhost') + MASTER_SERVER_ADDR = socket.gethostbyname(MASTER_SERVER) + DEPENDENCIES = ( + 'orchestra.contrib.orchestration', + 'orchestra.contrib.domains', + 'orchestra.contrib.websites', + 'orchestra.contrib.webapps', + 'orchestra.contrib.systemusers', + ) + + def add_route(self): + super(WebsiteMixin, self).add_route() + server = Server.objects.get() + backend = backends.apache.Apache2Controller.get_name() + Route.objects.get_or_create(backend=backend, match=True, host=server) + backend = Bind9MasterDomainController.get_name() + Route.objects.get_or_create(backend=backend, match=True, host=server) + + def validate_add_website(self, name, domain): + url = 'http://%s/%s' % (domain.name, self.page[0]) + self.assertEqual(self.page[2], requests.get(url).content) + + def test_add(self): + # TODO domains with "_" bad name! + domain_name = '%sdomain.lan' % random_ascii(10) + domain = Domain.objects.create(name=domain_name, account=self.account) + domain.records.create(type=Record.A, value=self.MASTER_SERVER_ADDR) + self.save_domain(domain) + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + website = '%s_website' % random_ascii(10) + self.add_website(website, domain, webapp) + self.addCleanup(self.delete_website, website) + self.validate_add_website(website, domain) + + +class RESTWebsiteMixin(RESTWebAppMixin): + @save_response_on_error + def save_domain(self, domain): + self.rest.domains.retrieve().get().save() + + @save_response_on_error + def add_website(self, name, domain, webapp, path='/'): + domain = self.rest.domains.retrieve(name=domain).get() + webapp = self.rest.webapps.retrieve(name=webapp).get() + contents = [{ + 'webapp': webapp, + 'path': path + }] + self.rest.websites.create(name=name, domains=[domain], contents=contents) + + @save_response_on_error + def delete_website(self, name): + self.rest.websites.retrieve(name=name).delete() + + @save_response_on_error + def add_content(self, website, webapp, path): + website = self.rest.websites.retrieve(name=website).get() + webapp = self.rest.webapps.retrieve(name=webapp).get() + website.contents.append({ + 'webapp': webapp, + 'path': path, + }) + website.save() + + # TODO test disable + # TODO test https (refactor ssl) + # TODO test php options + # TODO read php-version /fpm/fcgid + # TODO max_processes, timeouts, memory... + + +class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): + def test_mix_webapps(self): + domain_name = '%sdomain.lan' % random_ascii(10) + domain = Domain.objects.create(name=domain_name, account=self.account) + domain.records.create(type=Record.A, value=self.MASTER_SERVER_ADDR) + self.save_domain(domain) + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + website = '%s_website' % random_ascii(10) + self.add_website(website, domain, webapp) + self.addCleanup(self.delete_website, website) + self.validate_add_website(website, domain) + + self.type_value = PHPFPMWebAppMixin.type_value + self.backend = PHPFPMWebAppMixin.backend + self.page = PHPFPMWebAppMixin.page + self.add_route() + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + path = '/%s' % webapp + self.add_content(website, webapp, path) + url = 'http://%s%s/%s' % (domain.name, path, self.page[0]) + self.assertEqual(self.page[2], requests.get(url).content) + + self.type_value = PHPFPMWebAppMixin.type_value + self.backend = PHPFPMWebAppMixin.backend + self.page = PHPFPMWebAppMixin.page + self.add_route() + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + path = '/%s' % webapp + + self.add_content(website, webapp, path) + url = 'http://%s%s/%s' % (domain.name, path, self.page[0]) + self.assertEqual(self.page[2], requests.get(url).content) + + +class PHPFPMRESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): + pass + +#class AdminWebsiteTest(AdminWebsiteMixin, BaseLiveServerTestCase): +# pass + + + diff --git a/orchestra/contrib/websites/utils.py b/orchestra/contrib/websites/utils.py new file mode 100644 index 00000000..33f8dfee --- /dev/null +++ b/orchestra/contrib/websites/utils.py @@ -0,0 +1,5 @@ +def normurlpath(path): + if not path.startswith('/'): + path = '/' + path + path = path.rstrip('/') + return path.replace('//', '/') diff --git a/orchestra/contrib/websites/validators.py b/orchestra/contrib/websites/validators.py new file mode 100644 index 00000000..348a1cf0 --- /dev/null +++ b/orchestra/contrib/websites/validators.py @@ -0,0 +1,38 @@ +from django.core.exceptions import ValidationError +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + +from .models import Website + + +def validate_domain_protocol(website, domain, protocol): + if protocol == Website.HTTP: + qset = Q( + Q(protocol=Website.HTTP) | + Q(protocol=Website.HTTP_AND_HTTPS) | + Q(protocol=Website.HTTPS_ONLY) + ) + elif protocol == Website.HTTPS: + qset = Q( + Q(protocol=Website.HTTPS) | + Q(protocol=Website.HTTP_AND_HTTPS) | + Q(protocol=Website.HTTPS_ONLY) + ) + elif protocol in (Website.HTTP_AND_HTTPS, Website.HTTPS_ONLY): + qset = Q() + else: + raise ValidationError({ + 'protocol': _("Unknown protocol %s") % protocol + }) + if domain.websites.filter(qset).exclude(pk=website.pk).exists(): + raise ValidationError({ + 'domains': 'A website is already defined for "%s" on protocol %s' % (domain, protocol), + }) + + +def validate_server_name(domains): + if domains: + for domain in domains: + if not domain.name.startswith('*'): + return + raise ValidationError(_("At least one non-wildcard domain should be provided.")) diff --git a/orchestra/core/__init__.py b/orchestra/core/__init__.py new file mode 100644 index 00000000..9d540eee --- /dev/null +++ b/orchestra/core/__init__.py @@ -0,0 +1,52 @@ +from django.utils.text import format_lazy + +from ..utils.python import AttrDict + + +class Register(object): + def __init__(self, verbose_name=None): + self._registry = {} + self.verbose_name = verbose_name + + def __contains__(self, key): + return key in self._registry + + def __getitem__(self, key): + return self._registry[key] + + def __iter__(self): + return iter(self._registry.values()) + + def register(self, model, **kwargs): + if model in self._registry: + raise KeyError("%s already registered" % model) + if 'verbose_name' not in kwargs: + kwargs['verbose_name'] = model._meta.verbose_name + if 'verbose_name_plural' not in kwargs: + kwargs['verbose_name_plural'] = model._meta.verbose_name_plural + defaults = { + 'menu': True, + 'search': True, + 'model': model, + } + defaults.update(kwargs) + self._registry[model] = AttrDict(**defaults) + + def register_view(self, view_name, **kwargs): + if 'verbose_name' not in kwargs: + raise KeyError("%s verbose_name is required for views" % view_name) + if 'verbose_name_plural' not in kwargs: + kwargs['verbose_name_plural'] = format_lazy('{}' * 2, *[kwargs['verbose_name'], 's']) + + self.register(view_name, **kwargs) + + def get(self, *args): + if args: + return self._registry[args[0]] + return self._registry + + +services = Register(verbose_name='Services') +# TODO rename to something else +accounts = Register(verbose_name='Accounts') +administration = Register(verbose_name='Administration') diff --git a/orchestra/core/caches.py b/orchestra/core/caches.py new file mode 100644 index 00000000..ebf5f122 --- /dev/null +++ b/orchestra/core/caches.py @@ -0,0 +1,45 @@ +from threading import currentThread + +from django.core.cache.backends.dummy import DummyCache +from django.core.cache.backends.locmem import LocMemCache +from django.utils.deprecation import MiddlewareMixin + +_request_cache = {} + + +class RequestCache(LocMemCache): + """ LocMemCache is a threadsafe local memory cache """ + def __init__(self): + name = 'locmemcache@%i' % hash(currentThread()) + super(RequestCache, self).__init__(name, {}) + + +def get_request_cache(): + """ + Returns per-request cache when running RequestCacheMiddleware otherwise a + DummyCache instance (when running periodic tasks, tests or shell) + """ + try: + return _request_cache[currentThread()] + except KeyError: + return DummyCache('dummy', {}) + + +class RequestCacheMiddleware(MiddlewareMixin): + def process_request(self, request): + current_thread = currentThread() + cache = _request_cache.get(current_thread, RequestCache()) + _request_cache[current_thread] = cache + cache.clear() + + def clear_cache(self): + current_thread = currentThread() + if currentThread() in _request_cache: + _request_cache[current_thread].clear() + + def process_exception(self, request, exception): + self.clear_cache() + + def process_response(self, request, response): + self.clear_cache() + return response diff --git a/orchestra/core/context_processors.py b/orchestra/core/context_processors.py new file mode 100644 index 00000000..52db2046 --- /dev/null +++ b/orchestra/core/context_processors.py @@ -0,0 +1,9 @@ +from orchestra import settings + + +def site(request): + """ Adds site-related context variables to the context """ + return { + 'ORCHESTRA_SITE_NAME': settings.ORCHESTRA_SITE_NAME, + 'ORCHESTRA_SITE_VERBOSE_NAME': settings.ORCHESTRA_SITE_VERBOSE_NAME + } diff --git a/orchestra/core/translations.py b/orchestra/core/translations.py new file mode 100644 index 00000000..6bc7ebb6 --- /dev/null +++ b/orchestra/core/translations.py @@ -0,0 +1,13 @@ +class ModelTranslation(object): + """ + Collects all model fields that would be translated + + using 'makemessages --domain database' management command + """ + _registry = {} + + @classmethod + def register(cls, model, fields): + if model in cls._registry: + raise ValueError("Model %s already registered." % model.__name__) + cls._registry[model] = fields diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py new file mode 100644 index 00000000..f6e1fccd --- /dev/null +++ b/orchestra/core/validators.py @@ -0,0 +1,179 @@ +import logging +import re +from ipaddress import ip_address + +import phonenumbers +from django.core import validators +from django.core.exceptions import ValidationError +from django.utils.deconstruct import deconstructible +from django.utils.translation import gettext_lazy as _ + +from ..utils.python import import_class + + +logger = logging.getLogger(__name__) + + +def all_valid(*args): + """ helper function to merge multiple validators at once """ + if len(args) == 1: + # Dict + errors = {} + kwargs = args[0] + for field, validator in kwargs.items(): + try: + validator[0](*validator[1:]) + except ValidationError as error: + errors[field] = error + else: + # List + errors = [] + value, validators = args + for validator in validators: + try: + validator(value) + except ValidationError as error: + errors.append(error) + if errors: + raise ValidationError(errors) + + +@deconstructible +class OrValidator(object): + """ + Run validators with an OR logic + """ + def __init__(self, *validators): + self.validators = validators + + def __call__(self, value): + msg = [] + for validator in self.validators: + try: + validator(value) + except ValidationError as err: + msg.append(str(err)) + else: + return + raise ValidationError(' OR '.join(msg)) + + +def validate_ipv4_address(value): + msg = _("Not a valid IPv4 address") + try: + ip = ip_address(value) + except ValueError: + raise ValidationError(msg) + if ip.version != 4: + raise ValidationError(msg) + + +def validate_ipv6_address(value): + msg = _("Not a valid IPv6 address") + try: + ip = ip_address(value) + except ValueError: + raise ValidationError(msg) + if ip.version != 6: + raise ValidationError(msg) + + +def validate_ip_address(value): + msg = _("Not a valid IP address") + try: + ip_address(value) + except ValueError: + raise ValidationError(msg) + + +def validate_name(value): + """ + A single non-empty line of free-form text with no whitespace. + """ + validators.RegexValidator('^[\.\_\-0-9a-z]+$', + _("Enter a valid name (spaceless lowercase text including _.-)."), 'invalid')(value) + + +def validate_ascii(value): + try: + value.encode('ascii') + except UnicodeEncodeError: + raise ValidationError('This is not an ASCII string.') + + +def validate_hostname(hostname): + """ + Ensures that each segment + * contains at least one character and a maximum of 63 characters + * consists only of allowed characters + * doesn't begin or end with a hyphen. + http://stackoverflow.com/a/2532344 + """ + if len(hostname) > 255: + raise ValidationError(_("Too long for a hostname.")) + hostname = hostname.rstrip('.') + allowed = re.compile('(?!-)[A-Z\d-]{1,63}(? tag. + + Requires use of specific form support. (see ReadonlyForm or ReadonlyModelForm) + """ + + def __init__(self, *args, **kwargs): + kwargs['widget'] = kwargs.get('widget', SpanWidget) + super(SpanField, self).__init__(*args, **kwargs) diff --git a/orchestra/forms/options.py b/orchestra/forms/options.py new file mode 100644 index 00000000..5dfda313 --- /dev/null +++ b/orchestra/forms/options.py @@ -0,0 +1,98 @@ +from django import forms +from django.contrib.auth import forms as auth_forms +from django.utils.translation import gettext_lazy as _ + +from orchestra.utils.python import random_ascii + +from ..core.validators import validate_password + +from .fields import SpanField +from .widgets import SpanWidget + + +class UserCreationForm(forms.ModelForm): + """ + A form that creates a user, with no privileges, from the given username and + password. + """ + error_messages = { + 'password_mismatch': _("The two password fields didn't match."), + 'duplicate_username': _("A user with that username already exists."), + } + password1 = forms.CharField(label=_("Password"), + widget=forms.PasswordInput(attrs={'autocomplete': 'off'}), + validators=[validate_password]) + password2 = forms.CharField(label=_("Password confirmation"), + widget=forms.PasswordInput, + help_text=_("Enter the same password as above, for verification.")) + + def __init__(self, *args, **kwargs): + super(UserCreationForm, self).__init__(*args, **kwargs) + self.fields['password1'].help_text = _("Suggestion: %s") % random_ascii(10) + + def clean_password2(self): + password1 = self.cleaned_data.get('password1') + password2 = self.cleaned_data.get('password2') + if password1 and password2 and password1 != password2: + raise forms.ValidationError( + self.error_messages['password_mismatch'], + code='password_mismatch', + ) + return password2 + + def clean_username(self): + # Since model.clean() will check this, this is redundant, + # but it sets a nicer error message than the ORM and avoids conflicts with contrib.auth + username = self.cleaned_data["username"] + try: + self._meta.model._default_manager.get(username=username) + except self._meta.model.DoesNotExist: + return username + raise forms.ValidationError(self.error_messages['duplicate_username']) + + def save(self, commit=True): + user = super(UserCreationForm, self).save(commit=False) + user.set_password(self.cleaned_data['password1']) + if commit: + user.save() + return user + + +class UserChangeForm(forms.ModelForm): + password = auth_forms.ReadOnlyPasswordHashField(label=_("Password"), + help_text=_("Raw passwords are not stored, so there is no way to see " + "this user's password, but you can change it by " + "using this form. " + "Show hash.")) + + def clean_password(self): + # Regardless of what the user provides, return the initial value. + # This is done here, rather than on the field, because the + # field does not have access to the initial value + return self.initial["password"] + + +class NonStoredUserChangeForm(forms.ModelForm): + password = forms.CharField(label=_("Password"), required=False, + widget=SpanWidget(display='Unknown password'), + help_text=_("This service's password is not stored, so there is no way to see it, " + "but you can change it using this form.")) + + +class ReadOnlyFormMixin(object): + """ + Mixin class for ModelForm or Form that provides support for SpanField on readonly fields + Meta: + readonly_fields = (ro_field1, ro_field2) + """ + def __init__(self, *args, **kwargs): + super(ReadOnlyFormMixin, self).__init__(*args, **kwargs) + for name in self.Meta.readonly_fields: + field = self.fields[name] + if not isinstance(field, SpanField): + if not isinstance(field.widget, SpanWidget): + field.widget = SpanWidget() + original = self.initial.get(name) + if hasattr(self, 'instance'): + original = getattr(self.instance, name, original) + field.widget.original = original diff --git a/orchestra/forms/widgets.py b/orchestra/forms/widgets.py new file mode 100644 index 00000000..84e0f1ef --- /dev/null +++ b/orchestra/forms/widgets.py @@ -0,0 +1,73 @@ +import re +import textwrap + +from django import forms +from django.utils.safestring import mark_safe + +from django.templatetags.static import static + + +class SpanWidget(forms.Widget): + """ + Renders a value wrapped in a tag. + Requires use of specific form support. (see ReadonlyForm or ReadonlyModelForm) + """ + def __init__(self, *args, **kwargs): + self.tag = kwargs.pop('tag', '') + self.original = kwargs.pop('original', '') + self.display = kwargs.pop('display', None) + super(SpanWidget, self).__init__(*args, **kwargs) + + def render(self, name, value, attrs=None, renderer=None): + final_attrs = self.build_attrs(attrs, extra_attrs={'name':name}) + original = self.original or value + display = original if self.display is None else self.display + # Display icon + if isinstance(original, bool): + icon = static('admin/img/icon-%s.svg' % ('yes' if original else 'no',)) + return mark_safe('%s' % (icon, display)) + tag = self.tag[:-1] + endtag = '/'.join((self.tag[0], self.tag[1:])) + return mark_safe('%s%s >%s%s' % (tag, forms.utils.flatatt(final_attrs), display, endtag)) + + def value_from_datadict(self, data, files, name): + return self.original + + def _has_changed(self, initial, data): + return False + + +class PaddingCheckboxSelectMultiple(forms.CheckboxSelectMultiple): + """ Ugly hack to render this widget nicely on Django admin """ + def __init__(self, padding, attrs=None, choices=()): + super().__init__(attrs=attrs, choices=choices) + self.padding = padding + + def render(self, *args, **kwargs): + value = super().render(*args, **kwargs) + value = re.sub(r'^