pangea/django-orchestra
7
0
Fork 0
Code Issues 1 Pull requests 2 Packages Projects Releases Wiki Activity

Initial lists tests

Browse source
This commit is contained in:
Marc 2014-10-09 17:04:12 +00:00
parent e124c830ac
commit 831347fb03
33 changed files with 650 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
TODO.md
View file

@ -166,3 +166,7 @@ APPS app?
* disable account triggers save on cascade to execute backends save(update_field=[])
* validate database user names
* multiple domains creation; line separated domains

1
orchestra/admin/forms.py
View file

@ -115,7 +115,6 @@ class AdminPasswordChangeForm(forms.Form):
for ix, rel in enumerate(self.related):
password = self.cleaned_data['password1_%s' % ix]
if password:
print password
set_password = getattr(rel, 'set_password')
set_password(password)
if commit:

33
orchestra/admin/utils.py
View file

@ -1,4 +1,5 @@
import datetime
import inspect
from functools import wraps
from django.conf import settings
@ -31,30 +32,26 @@ def get_modeladmin(model, import_module=True):
return get_modeladmin(model, import_module=False)
def insertattr(model, name, value, weight=0):
def insertattr(model, name, value):
""" Inserts attribute to a modeladmin """
modeladmin_class = model
modeladmin = None
if models.Model in model.__mro__:
modeladmin_class = type(get_modeladmin(model))
modeladmin = get_modeladmin(model)
modeladmin_class = type(modeladmin)
elif not inspect.isclass(model):
modeladmin = model
modeladmin_class = type(modeladmin)
else:
modeladmin_class = model
# Avoid inlines defined on parent class be shared between subclasses
# Seems that if we use tuples they are lost in some conditions like changing
# the tuple in modeladmin.__init__
if not getattr(modeladmin_class, name):
setattr(modeladmin_class, name, [])
inserted_attrs = getattr(modeladmin_class, '__inserted_attrs__', {})
if not name in inserted_attrs:
weights = {}
if hasattr(modeladmin_class, 'weights') and name in modeladmin_class.weights:
weights = modeladmin_class.weights.get(name)
inserted_attrs[name] = [
(attr, weights.get(attr, 0)) for attr in getattr(modeladmin_class, name)
]
inserted_attrs[name].append((value, weight))
inserted_attrs[name].sort(key=lambda a: a[1])
setattr(modeladmin_class, name, [ attr[0] for attr in inserted_attrs[name] ])
setattr(modeladmin_class, '__inserted_attrs__', inserted_attrs)
setattr(modeladmin_class, name, list(getattr(modeladmin_class, name))+[value])
if modeladmin:
# make sure class and object share the same attribute, to avoid wierd bugs
setattr(modeladmin, name, getattr(modeladmin_class, name))
def wrap_admin_view(modeladmin, view):
@ -84,7 +81,7 @@ def action_to_view(action, modeladmin):
response = action(modeladmin, request, queryset)
if not response:
opts = modeladmin.model._meta
url = 'admin:%s_%s_change' % (opts.app_label, opts.module_name)
url = 'admin:%s_%s_change' % (opts.app_label, opts.model_name)
return redirect(url, object_id)
return response
return action_view

2
orchestra/apps/databases/admin.py
View file

@ -89,7 +89,7 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
if not change:
user = form.cleaned_data['user']
if not user:
user = DatabaseUser.objects.create(
user = DatabaseUser(
username=form.cleaned_data['username'],
type=obj.type,
account_id = obj.account.pk,

18
orchestra/apps/databases/backends.py
View file

@ -41,28 +41,28 @@ class MySQLUserBackend(ServiceController):
verbose_name = "MySQL user"
model = 'databases.DatabaseUser'
def save(self, database):
if database.type == database.MYSQL:
context = self.get_context(database)
def save(self, user):
if user.type == user.MYSQL:
context = self.get_context(user)
self.append(
"mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";'" % context
"mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context
)
self.append(
"mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
" WHERE User=\"%(username)s\";'" % context
)
def delete(self, database):
if database.type == database.MYSQL:
def delete(self, user):
if user.type == user.MYSQL:
context = self.get_context(database)
self.append(
"mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context
)
def get_context(self, database):
def get_context(self, user):
return {
'username': database.username,
'password': database.password,
'username': user.username,
'password': user.password,
'host': settings.DATABASES_DEFAULT_HOST,
}

26
orchestra/apps/databases/forms.py
View file

@ -30,9 +30,9 @@ class DatabaseUserCreationForm(forms.ModelForm):
def save(self, commit=True):
user = super(DatabaseUserCreationForm, self).save(commit=False)
user.set_password(self.cleaned_data["password1"])
if commit:
user.save()
# user.set_password(self.cleaned_data["password1"])
# if commit:
# user.save()
return user
@ -89,16 +89,16 @@ class DatabaseCreationForm(DatabaseUserCreationForm):
def save(self, commit=True):
db = super(DatabaseUserCreationForm, self).save(commit=False)
user = self.cleaned_data['user']
if commit:
if not user:
user = DatabaseUser(
username=self.cleaned_data['username'],
type=self.cleaned_data['type'],
)
user.set_password(self.cleaned_data["password1"])
user.save()
role, __ = Role.objects.get_or_create(database=db, user=user)
# if commit:
# user = self.cleaned_data['user']
# if not user:
# user = DatabaseUser(
# username=self.cleaned_data['username'],
# type=self.cleaned_data['type'],
# )
# user.set_password(self.cleaned_data["password1"])
# user.save()
# role, __ = Role.objects.get_or_create(database=db, user=user)
return db

11
orchestra/apps/databases/models.py
View file

@ -13,7 +13,7 @@ class Database(models.Model):
MYSQL = 'mysql'
POSTGRESQL = 'postgresql'
name = models.CharField(_("name"), max_length=128,
name = models.CharField(_("name"), max_length=64, # MySQL limit
validators=[validators.validate_name])
users = models.ManyToManyField('databases.DatabaseUser',
verbose_name=_("users"),
@ -53,9 +53,7 @@ class Role(models.Model):
msg = _("Database and user type doesn't match")
raise validators.ValidationError(msg)
roles = self.database.roles.values('id')
print roles
if not roles or (len(roles) == 1 and roles[0].id == self.id):
print 'seld'
self.is_owner = True
@ -63,9 +61,9 @@ class DatabaseUser(models.Model):
MYSQL = 'mysql'
POSTGRESQL = 'postgresql'
username = models.CharField(_("username"), max_length=128,
username = models.CharField(_("username"), max_length=16, # MySQL usernames 16 char long
validators=[validators.validate_name])
password = models.CharField(_("password"), max_length=128)
password = models.CharField(_("password"), max_length=256)
type = models.CharField(_("type"), max_length=32,
choices=settings.DATABASES_TYPE_CHOICES,
default=settings.DATABASES_DEFAULT_TYPE)
@ -87,8 +85,7 @@ class DatabaseUser(models.Model):
# MySQL stores sha1(sha1(password).binary).hex
binary = hashlib.sha1(password).digest()
hexdigest = hashlib.sha1(binary).hexdigest()
password = '*%s' % hexdigest.upper()
self.password = password
self.password = '*%s' % hexdigest.upper()
else:
raise TypeError("Database type '%s' not supported" % self.type)

6
orchestra/apps/databases/serializers.py
View file

@ -8,7 +8,7 @@ from orchestra.core.validators import validate_password
from .models import Database, DatabaseUser, Role
class UserSerializer(serializers.HyperlinkedModelSerializer):
class UserRoleSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = Role
fields = ('user', 'is_owner',)
@ -21,11 +21,11 @@ class RoleSerializer(serializers.HyperlinkedModelSerializer):
class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
users = UserSerializer(source='roles', many=True)
roles = UserRoleSerializer(many=True)
class Meta:
model = Database
fields = ('url', 'name', 'type', 'users')
fields = ('url', 'name', 'type', 'roles')
class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):

13
orchestra/apps/databases/tests/functional_tests/tests.py
View file

@ -1,5 +1,6 @@
import MySQLdb
import os
import time
from functools import partial
from django.conf import settings as djsettings
@ -52,7 +53,7 @@ class DatabaseTestMixin(object):
def test_add(self):
dbname = '%s_database' % random_ascii(5)
username = '%s_dbuser' % random_ascii(10)
username = '%s_dbuser' % random_ascii(5)
password = '@!?%spppP001' % random_ascii(5)
self.add(dbname, username, password)
self.validate_create_table(dbname, username, password)
@ -61,6 +62,10 @@ class DatabaseTestMixin(object):
class MySQLBackendMixin(object):
db_type = 'mysql'
def setUp(self):
super(MySQLBackendMixin, self).setUp()
settings.DATABASES_DEFAULT_HOST = '10.228.207.207'
def add_route(self):
server = Server.objects.create(name=self.MASTER_SERVER)
backend = backends.MySQLBackend.get_name()
@ -73,14 +78,13 @@ class MySQLBackendMixin(object):
def validate_create_table(self, name, username, password):
db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name)
cur = db.cursor()
cur.execute('CREATE TABLE test;')
cur.execute('CREATE TABLE test ( id INT ) ;')
def validate_delete(self, name, username, password):
self.asseRaises(MySQLdb.ConnectionError,
self.validate_create_table, name, username, password)
class RESTDatabaseMixin(DatabaseTestMixin):
def setUp(self):
super(RESTDatabaseMixin, self).setUp()
@ -89,7 +93,8 @@ class RESTDatabaseMixin(DatabaseTestMixin):
@save_response_on_error
def add(self, dbname, username, password):
user = self.rest.databaseusers.create(username=username, password=password)
self.rest.databases.create(name=dbname, user=user, type=self.db_type)
# TODO fucking nested objects
self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type)
class AdminDatabaseMixin(DatabaseTestMixin):

81
orchestra/apps/lists/backends.py
View file

@ -6,12 +6,93 @@ from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from . import settings
from .models import List
class MailmanBackend(ServiceController):
verbose_name = "Mailman"
model = 'lists.List'
def include_virtual_alias_domain(self, context):
if context['address_domain']:
self.append(textwrap.dedent("""
[[ $(grep "^\s*%(address_domain)s\s*$" %(virtual_alias_domains)s) ]] || {
echo "%(address_domain)s" >> %(virtual_alias_domains)s
UPDATED_VIRTUAL_ALIAS_DOMAINS=1
}""" % context
))
def exclude_virtual_alias_domain(self, context):
address_domain = context['address_domain']
if not List.objects.filter(address_domain=address_domain).exists():
self.append('sed -i "/^%(address_domain)s\s*/d" %(virtual_alias_domains)s' % context)
def get_virtual_aliases(self, context):
aliases = []
addresses = [
'',
'-admin',
'-bounces',
'-confirm',
'-join',
'-leave',
'-owner',
'-request',
'-subscribe',
'-unsubscribe'
]
for address in addresses:
context['address'] = address
aliases.append("%(address_name)s%(address)s@%(domain)s\t%(name)s%(address)s" % context)
return '\n'.join(aliases)
def save(self, mail_list):
if not getattr(mail_list, 'password', None):
# TODO
# Create only support for now
return
context = self.get_context(mail_list)
self.append("newlist --quiet --emailhost='%(domain)s' '%(name)s' '%(admin)s' '%(password)s'" % context)
if mail_list.address:
context['aliases'] = self.get_virtual_aliases(context)
self.append(
"if [[ ! $(grep '^\s*%(name)s\s' %(virtual_alias)s) ]]; then\n"
" echo '# %(banner)s\n%(aliases)s\n' >> %(virtual_alias)s\n"
" UPDATED_VIRTUAL_ALIAS=1\n"
"fi" % context
)
self.include_virtual_alias_domain(context)
def delete(self, mail_list):
pass
def commit(self):
context = self.get_context_files()
self.append(textwrap.dedent("""
[[ $UPDATED_VIRTUAL_ALIAS == 1 ]] && { postmap %(virtual_alias)s; }
[[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; }
""" % context
))
def get_context_files(self):
return {
'virtual_alias': settings.LISTS_VIRTUAL_ALIAS_PATH,
'virtual_alias_domains': settings.MAILS_VIRTUAL_ALIAS_DOMAINS_PATH,
}
def get_context(self, mail_list):
context = self.get_context_files()
context.update({
'banner': self.get_banner(),
'name': mail_list.name,
'password': mail_list.password,
'domain': mail_list.address_domain or settings.LISTS_DEFAULT_DOMAIN,
'address_name': mail_list.address_name,
'address_domain': mail_list.address_domain,
'admin': mail_list.admin_email,
})
return context
class MailmanTraffic(ServiceMonitor):
model = 'lists.List'

14
orchestra/apps/lists/models.py
View file

@ -7,9 +7,11 @@ from orchestra.core.validators import validate_name
from . import settings
# TODO address and domain, perhaps allow only domain?
class List(models.Model):
name = models.CharField(_("name"), max_length=128, unique=True,
validators=[validate_name])
name = models.CharField(_("name"), max_length=128, unique=True, validators=[validate_name],
help_text=_("Default list address <name>@%s") % settings.LISTS_DEFAULT_DOMAIN)
address_name = models.CharField(_("address name"), max_length=128,
validators=[validate_name], blank=True)
address_domain = models.ForeignKey(settings.LISTS_DOMAIN_MODEL,
@ -23,7 +25,13 @@ class List(models.Model):
unique_together = ('address_name', 'address_domain')
def __unicode__(self):
return "%s@%s" % (self.address_name, self.address_domain)
return self.name
@property
def address(self):
if self.address_name and self.address_domain:
return "%s@%s" % (self.address_name, self.address_domain)
return ''
def get_username(self):
return self.name

25
orchestra/apps/lists/serializers.py
View file

@ -1,11 +1,34 @@
from django.forms import widgets
from django.utils.translation import ugettext, ugettext_lazy as _
from rest_framework import serializers
from orchestra.apps.accounts.serializers import AccountSerializerMixin
from orchestra.core.validators import validate_password
from .models import List
# TODO create PasswordSerializerMixin
class ListSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
password = serializers.CharField(max_length=128, label=_('Password'),
validators=[validate_password], write_only=True, required=False,
widget=widgets.PasswordInput)
class Meta:
model = List
fields = ('url', 'name', 'address_name', 'address_domain')
fields = ('url', 'name', 'address_name', 'address_domain', 'admin_email')
def validate_password(self, attrs, source):
""" POST only password """
if self.object:
if 'password' in attrs:
raise serializers.ValidationError(_("Can not set password"))
elif 'password' not in attrs:
raise serializers.ValidationError(_("Password required"))
return attrs
def save_object(self, obj, **kwargs):
if not obj.pk:
obj.set_password(self.init_data.get('password', ''))
super(ListSerializer, self).save_object(obj, **kwargs)

13
orchestra/apps/lists/settings.py
View file

@ -1,11 +1,20 @@
from django.conf import settings
# Data access
LISTS_DOMAIN_MODEL = getattr(settings, 'LISTS_DOMAIN_MODEL', 'domains.Domain')
LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'grups.orchestra.lan')
LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'lists.orchestra.lan')
LISTS_MAILMAN_POST_LOG_PATH = getattr(settings, 'LISTS_MAILMAN_POST_LOG_PATH',
'/var/log/mailman/post')
LISTS_VIRTUAL_ALIAS_PATH = getattr(settings, 'LISTS_VIRTUAL_ALIAS_PATH',
'/etc/postfix/mailman_virtual_aliases')
MAILS_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_DOMAINS_PATH',
'/etc/postfix/mailman_virtual_domains')

0
orchestra/apps/lists/tests/__init__.py Normal file
View file

0
orchestra/apps/lists/tests/functional_tests/__init__.py Normal file
View file

158
orchestra/apps/lists/tests/functional_tests/tests.py Normal file
View file

@ -0,0 +1,158 @@
import email.utils
import os
import smtplib
import time
import textwrap
from email.mime.text import MIMEText
from django.conf import settings as djsettings
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import CommandError
from django.core.urlresolvers import reverse
from selenium.webdriver.support.select import Select
from orchestra.apps.accounts.models import Account
from orchestra.apps.domains.models import Domain
from orchestra.apps.orchestration.models import Server, Route
from orchestra.apps.resources.models import Resource
from orchestra.utils.system import run, sshrun
from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error, save_response_on_error
from ... import backends, settings
from ...models import List
class ListMixin(object):
MASTER_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost')
DEPENDENCIES = (
'orchestra.apps.orchestration',
'orchestra.apps.domains',
'orchestra.apps.lists',
)
def setUp(self):
super(ListMixin, self).setUp()
self.add_route()
djsettings.DEBUG = True
def validate_add(self, name, address=None):
sshrun(self.MASTER_SERVER, 'list_members %s' % name, display=False)
if not address:
address = "%s@%s" % (name, settings.LISTS_DEFAULT_DOMAIN)
subscribe_address = "{}-subscribe@{}".format(*address.split('@'))
self.subscribe(subscribe_address)
time.sleep(2)
sshrun(self.MASTER_SERVER,
'grep -v ":\|^\s\|^$\|-\|\.\|\s" /var/spool/mail/nobody | base64 -d | grep "%s"' % address, display=False)
def subscribe(self, subscribe_address):
msg = MIMEText('')
msg['To'] = subscribe_address
msg['From'] = 'root@%s' % self.MASTER_SERVER
msg['Subject'] = 'subscribe'
server = smtplib.SMTP(self.MASTER_SERVER, 25)
try:
server.ehlo()
server.starttls()
server.ehlo()
server.sendmail(msg['From'], msg['To'], msg.as_string())
finally:
server.quit()
def add_route(self):
server = Server.objects.create(name=self.MASTER_SERVER)
backend = backends.MailmanBackend.get_name()
Route.objects.create(backend=backend, match=True, host=server)
def atest_add(self):
name = '%s_list' % random_ascii(10)
password = '@!?%spppP001' % random_ascii(5)
admin_email = 'root@test3.orchestra.lan'
self.add(name, password, admin_email)
self.validate_add(name)
# self.addCleanup(self.delete, username)
def test_add_with_address(self):
name = '%s_list' % random_ascii(10)
password = '@!?%spppP001' % random_ascii(5)
print password
admin_email = 'root@test3.orchestra.lan'
address_name = '%s_name' % random_ascii(10)
domain_name = '%sdomain.lan' % random_ascii(10)
address_domain = Domain.objects.create(name=domain_name, account=self.account)
self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain)
self.validate_add(name, address="%s@%s" % (address_name, address_domain))
class RESTListMixin(ListMixin):
def setUp(self):
super(RESTListMixin, self).setUp()
self.rest_login()
@save_response_on_error
def add(self, name, password, admin_email, address_name=None, address_domain=None):
extra = {}
if address_name:
extra.update({
'address_name': address_name,
'address_domain': self.rest.domains.retrieve(name=address_domain.name).get().url,
})
self.rest.lists.create(name=name, password=password, admin_email=admin_email, **extra)
@save_response_on_error
def delete(self, username):
list = self.rest.lists.retrieve(name=username).get()
list.delete()
class AdminListMixin(ListMixin):
def setUp(self):
super(AdminListMixin, self).setUp()
self.admin_login()
@snapshot_on_error
def add(self, name, password, admin_email):
url = self.live_server_url + reverse('admin:mails_List_add')
self.selenium.get(url)
account_input = self.selenium.find_element_by_id('id_account')
account_select = Select(account_input)
account_select.select_by_value(str(self.account.pk))
name_field = self.selenium.find_element_by_id('id_name')
name_field.send_keys(username)
password_field = self.selenium.find_element_by_id('id_password1')
password_field.send_keys(password)
password_field = self.selenium.find_element_by_id('id_password2')
password_field.send_keys(password)
if quota is not None:
quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
quota_field = self.selenium.find_element_by_id(quota_id)
quota_field.clear()
quota_field.send_keys(quota)
if filtering is not None:
filtering_input = self.selenium.find_element_by_id('id_filtering')
filtering_select = Select(filtering_input)
filtering_select.select_by_value("CUSTOM")
filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
filtering_inline.click()
time.sleep(0.5)
filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
filtering_field.send_keys(filtering)
name_field.submit()
self.assertNotEqual(url, self.selenium.current_url)
class RESTListTest(RESTListMixin, BaseLiveServerTestCase):
pass
#class AdminListTest(AdminListMixin, BaseLiveServerTestCase):
# pass

29
orchestra/apps/mails/admin.py
View file

@ -9,10 +9,9 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
from orchestra.admin.utils import admin_link, change_url
from orchestra.apps.accounts.admin import SelectAccountAdminMixin, AccountAdminMixin
from orchestra.forms import UserCreationForm, UserChangeForm
from .filters import HasMailboxListFilter, HasForwardListFilter, HasAddressListFilter
from .forms import MailboxCreationForm, AddressForm
from .forms import MailboxCreationForm, MailboxChangeForm, AddressForm
from .models import Mailbox, Address, Autoresponse
@ -28,36 +27,34 @@ class AutoresponseInline(admin.StackedInline):
class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
list_display = (
'name', 'account_link', 'uses_custom_filtering', 'display_addresses'
'name', 'account_link', 'filtering', 'display_addresses'
)
list_filter = (HasAddressListFilter,)
list_filter = (HasAddressListFilter, 'filtering')
add_fieldsets = (
(None, {
'fields': ('account', 'name', 'password1', 'password2'),
'fields': ('account', 'name', 'password1', 'password2', 'filtering'),
}),
(_("Filtering"), {
(_("Custom filtering"), {
'classes': ('collapse',),
'fields': ('custom_filtering',),
}),
)
fieldsets = (
(None, {
'classes': ('wide',),
'fields': ('name', 'password', 'is_active', 'account_link'),
'fields': ('name', 'password', 'is_active', 'account_link', 'filtering'),
}),
(_("Filtering"), {
(_("Custom filtering"), {
'classes': ('collapse',),
'fields': ('custom_filtering',),
}),
(_("Addresses"), {
'classes': ('wide',),
'fields': ('addresses_field',)
}),
)
readonly_fields = ('account_link', 'display_addresses', 'addresses_field')
change_readonly_fields = ('name',)
add_form = MailboxCreationForm
form = UserChangeForm
form = MailboxChangeForm
def display_addresses(self, mailbox):
addresses = []
@ -68,16 +65,10 @@ class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdm
display_addresses.short_description = _("Addresses")
display_addresses.allow_tags = True
def uses_custom_filtering(self, mailbox):
return bool(mailbox.custom_filtering)
uses_custom_filtering.short_description = _("Custom filter")
uses_custom_filtering.boolean = True
uses_custom_filtering.admin_order_field = 'custom_filtering'
def get_fieldsets(self, request, obj=None):
""" not collapsed filtering when exists """
fieldsets = super(MailboxAdmin, self).get_fieldsets(request, obj=obj)
if obj and obj.custom_filtering:
if obj and obj.filtering == obj.CUSTOM:
fieldsets = copy.deepcopy(fieldsets)
fieldsets[1][1]['classes'] = fieldsets[0][1]['fields'] + ('open',)
return fieldsets
@ -97,7 +88,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdm
name = '%s@%s' % (name, domain)
value += '<li><a href="%s">%s</a></li>' % (url, name)
value = '<ul>%s</ul>' % value
return mark_safe('<div style="padding-left: 100px;">%s</div>' % value)
return mark_safe('<div style="padding-left: 10px;">%s</div>' % value)
addresses_field.short_description = _("Addresses")
addresses_field.allow_tags = True

131
orchestra/apps/mails/backends.py
View file

@ -1,3 +1,4 @@
import logging
import textwrap
import os
@ -13,11 +14,12 @@ from .models import Address
# TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
# TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
# TODO Set first/last_valid_uid/gid settings to contain only the range actually used by mail processes
# TODO Insert "/./" inside the returned home directory, eg.: home=/home/./user to chroot into /home, or home=/home/user/./ to chroot into /home/user.
# TODO mount the filesystem with "nosuid" option
logger = logging.getLogger(__name__)
class PasswdVirtualUserBackend(ServiceController):
verbose_name = _("Mail virtual user (passwd-file)")
model = 'mails.Mailbox'
@ -36,22 +38,27 @@ class PasswdVirtualUserBackend(ServiceController):
self.append("mkdir -p %(home)s" % context)
self.append("chown %(uid)s.%(gid)s %(home)s" % context)
def set_mailbox(self, context):
self.append(textwrap.dedent("""
if [[ ! $(grep "^%(username)s@%(mailbox_domain)s\s" %(virtual_mailbox_maps)s) ]]; then
echo "%(username)s@%(mailbox_domain)s\tOK" >> %(virtual_mailbox_maps)s
UPDATED_VIRTUAL_MAILBOX_MAPS=1
fi""" % context))
def generate_filter(self, mailbox, context):
now = timezone.now().strftime("%B %d, %Y, %H:%M")
context['filtering'] = (
"# Sieve Filter\n"
"# Generated by Orchestra %s\n\n" % now
)
if mailbox.custom_filtering:
context['filtering'] += mailbox.custom_filtering
self.append("doveadm mailbox create -u %(username)s Spam" % context) # TODO override webmail filters???
context['filtering_path'] = os.path.join(context['home'], '.dovecot.sieve')
filtering = mailbox.get_filtering()
if filtering:
context['filtering'] = '# %(banner)s\n' + filtering
self.append("echo '%(filtering)s' > %(filtering_path)s" % context)
else:
context['filtering'] += settings.MAILS_DEFAUL_FILTERING
context['filter_path'] = os.path.join(context['home'], '.orchestra.sieve')
self.append("echo '%(filtering)s' > %(filter_path)s" % context)
self.append("rm -f %(filtering_path)s" % context)
def save(self, mailbox):
context = self.get_context(mailbox)
self.set_user(context)
self.set_mailbox(context)
self.generate_filter(mailbox, context)
def delete(self, mailbox):
@ -59,6 +66,8 @@ class PasswdVirtualUserBackend(ServiceController):
self.append("{ sleep 2 && killall -u %(uid)s -s KILL; } &" % context)
self.append("killall -u %(uid)s || true" % context)
self.append("sed -i '/^%(username)s:.*/d' %(passwd_path)s" % context)
self.append("sed -i '/^%(username)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s" % context)
self.append("UPDATED_VIRTUAL_MAILBOX_MAPS=1")
# TODO delete
context['deleted'] = context['home'].rstrip('/') + '.deleted'
self.append("mv %(home)s %(deleted)s" % context)
@ -75,6 +84,15 @@ class PasswdVirtualUserBackend(ServiceController):
unit = mailbox.resources.disk.unit[0].upper()
return 'userdb_quota_rule=*:bytes=%i%s' % (quota, unit)
def commit(self):
context = {
'virtual_mailbox_maps': settings.MAILS_VIRTUAL_MAILBOX_MAPS_PATH
}
self.append(
"[[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && { postmap %(virtual_mailbox_maps)s; }"
% context
)
def get_context(self, mailbox):
context = {
'name': mailbox.name,
@ -86,9 +104,12 @@ class PasswdVirtualUserBackend(ServiceController):
'quota': self.get_quota(mailbox),
'passwd_path': settings.MAILS_PASSWD_PATH,
'home': mailbox.get_home(),
'banner': self.get_banner(),
'virtual_mailbox_maps': settings.MAILS_VIRTUAL_MAILBOX_MAPS_PATH,
'mailbox_domain': settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
}
context['extra_fields'] = self.get_extra_fields(mailbox, context)
context['passwd'] = '{username}:{password}:{uid}:{gid}:,,,:{home}:{extra_fields}'.format(**context)
context['passwd'] = '{username}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context)
return context
@ -96,62 +117,76 @@ class PostfixAddressBackend(ServiceController):
verbose_name = _("Postfix address")
model = 'mails.Address'
def include_virtdomain(self, context):
self.append(
'[[ $(grep "^\s*%(domain)s\s*$" %(virtdomains)s) ]]'
' || { echo "%(domain)s" >> %(virtdomains)s; UPDATED_VIRTDOMAINS=1; }' % context
)
def exclude_virtdomain(self, context):
domain = context['domain']
if not Address.objects.filter(domain=domain).exists():
self.append('sed -i "s/^%(domain)s//" %(virtdomains)s' % context)
def update_virtusertable(self, context):
def include_virtual_alias_domain(self, context):
self.append(textwrap.dedent("""
LINE="%(email)s\t%(destination)s"
if [[ ! $(grep "^%(email)s\s" %(virtusertable)s) ]]; then
echo "${LINE}" >> %(virtusertable)s
UPDATED_VIRTUSERTABLE=1
else
if [[ ! $(grep "^${LINE}$" %(virtusertable)s) ]]; then
sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtusertable)s
UPDATED_VIRTUSERTABLE=1
fi
fi""" % context
[[ $(grep "^\s*%(domain)s\s*$" %(virtual_alias_domains)s) ]] || {
echo "%(domain)s" >> %(virtual_alias_domains)s
UPDATED_VIRTUAL_ALIAS_DOMAINS=1
}""" % context
))
def exclude_virtusertable(self, context):
def exclude_virtual_alias_domain(self, context):
domain = context['domain']
if not Address.objects.filter(domain=domain).exists():
self.append('sed -i "/^%(domain)s\s*/d" %(virtual_alias_domains)s' % context)
def update_virtual_alias_maps(self, address, context):
destination = []
for mailbox in address.get_mailboxes():
context['mailbox'] = mailbox
destination.append("%(mailbox)s@%(mailbox_domain)s" % context)
for forward in address.forward:
if '@' in forward:
destination.append(forward)
if destination:
context['destination'] = ' '.join(destination)
self.append(textwrap.dedent("""
LINE="%(email)s\t%(destination)s"
if [[ ! $(grep "^%(email)s\s" %(virtual_alias_maps)s) ]]; then
echo "${LINE}" >> %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
else
if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then
sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
fi
fi""" % context
))
else:
logger.warning("Address %i is empty" % address.pk)
self.append('sed -i "/^%(email)s\s/d" %(virtual_alias_maps)s')
self.append('UPDATED_VIRTUAL_ALIAS_MAPS=1')
def exclude_virtual_alias_maps(self, context):
self.append(textwrap.dedent("""
if [[ $(grep "^%(email)s\s") ]]; then
sed -i "s/^%(email)s\s.*$//" %(virtusertable)s
UPDATED=1
sed -i "/^%(email)s\s.*$/d" %(virtual_alias_maps)s
UPDATED_VIRTUAL_ALIAS_MAPS=1
fi"""
))
def save(self, address):
context = self.get_context(address)
self.include_virtdomain(context)
self.update_virtusertable(context)
self.include_virtual_alias_domain(context)
self.update_virtual_alias_maps(address, context)
def delete(self, address):
context = self.get_context(address)
self.exclude_virtdomain(context)
self.exclude_virtusertable(context)
self.exclude_virtual_alias_domain(context)
self.exclude_virtual_alias_maps(context)
def commit(self):
context = self.get_context_files()
self.append(textwrap.dedent("""
[[ $UPDATED_VIRTUSERTABLE == 1 ]] && { postmap %(virtusertable)s; }
# TODO not sure if always needed
[[ $UPDATED_VIRTDOMAINS == 1 ]] && { /etc/init.d/postfix reload; }
[[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; }
[[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; }
""" % context
))
def get_context_files(self):
return {
'virtdomains': settings.MAILS_VIRTDOMAINS_PATH,
'virtusertable': settings.MAILS_VIRTUSERTABLE_PATH,
'virtual_alias_domains': settings.MAILS_VIRTUAL_ALIAS_DOMAINS_PATH,
'virtual_alias_maps': settings.MAILS_VIRTUAL_ALIAS_MAPS_PATH
}
def get_context(self, address):
@ -159,7 +194,7 @@ class PostfixAddressBackend(ServiceController):
context.update({
'domain': address.domain,
'email': address.email,
'destination': address.destination,
'mailbox_domain': settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
})
return context

18
orchestra/apps/mails/forms.py
View file

@ -1,9 +1,23 @@
from django import forms
from django.utils.translation import ugettext_lazy as _
from orchestra.forms import UserCreationForm
from orchestra.forms import UserCreationForm, UserChangeForm
class MailboxCreationForm(UserCreationForm):
class CleanCustomFilteringMixin(object):
def clean_custom_filtering(self):
filtering = self.cleaned_data['filtering']
custom_filtering = self.cleaned_data['custom_filtering']
if filtering == self._meta.model.CUSTOM and not custom_filtering:
raise forms.ValidationError(_("You didn't provide any custom filtering"))
return custom_filtering
class MailboxChangeForm(CleanCustomFilteringMixin, UserChangeForm):
pass
class MailboxCreationForm(CleanCustomFilteringMixin, UserCreationForm):
def clean_name(self):
# Since model.clean() will check this, this is redundant,
# but it sets a nicer error message than the ORM and avoids conflicts with contrib.auth

56
orchestra/apps/mails/models.py
View file

@ -11,6 +11,8 @@ from . import validators, settings
# TODO rename app to mailboxes
class Mailbox(models.Model):
CUSTOM = 'CUSTOM'
name = models.CharField(_("name"), max_length=64, unique=True,
help_text=_("Required. 30 characters or fewer. Letters, digits and "
"@/./+/-/_ only."),
@ -19,6 +21,9 @@ class Mailbox(models.Model):
password = models.CharField(_("password"), max_length=128)
account = models.ForeignKey('accounts.Account', verbose_name=_("account"),
related_name='mailboxes')
filtering = models.CharField(max_length=16,
choices=[(k, v[0]) for k,v in settings.MAILS_MAILBOX_FILTERINGS.iteritems()],
default=settings.MAILS_MAILBOX_DEFAULT_FILTERING)
custom_filtering = models.TextField(_("filtering"), blank=True,
validators=[validators.validate_sieve],
help_text=_("Arbitrary email filtering in sieve language. "
@ -52,6 +57,28 @@ class Mailbox(models.Model):
home = settings.MAILS_HOME % context
return home.rstrip('/')
def clean(self):
if self.custom_filtering and self.filtering != self.CUSTOM:
self.custom_filtering = ''
def get_filtering(self):
__, filtering = settings.MAILS_MAILBOX_FILTERINGS[self.filtering]
if isinstance(filtering, basestring):
return filtering
return filtering(self)
def delete(self, *args, **kwargs):
super(Mailbox, self).delete(*args, **kwargs)
# Cleanup related addresses
for address in Address.objects.filter(forward__regex=r'.*(^|\s)+%s($|\s)+.*' % self.name):
forward = address.forward.split()
forward.remove(self.name)
address.forward = ' '.join(forward)
if not address.destination:
address.delete()
else:
address.save()
class Address(models.Model):
name = models.CharField(_("name"), max_length=64,
@ -63,7 +90,8 @@ class Address(models.Model):
verbose_name=_("mailboxes"),
related_name='addresses', blank=True)
forward = models.CharField(_("forward"), max_length=256, blank=True,
validators=[validators.validate_forward], help_text=_("Space separated email addresses"))
validators=[validators.validate_forward],
help_text=_("Space separated email addresses or mailboxes"))
account = models.ForeignKey('accounts.Account', verbose_name=_("Account"),
related_name='addresses')
@ -78,12 +106,26 @@ class Address(models.Model):
def email(self):
return "%s@%s" % (self.name, self.domain)
@property
def destination(self):
destinations = list(self.mailboxes.values_list('name', flat=True))
if self.forward:
destinations.append(self.forward)
return ' '.join(destinations)
# @property
# def destination(self):
# destinations = list(self.mailboxes.values_list('name', flat=True))
# if self.forward:
# destinations.append(self.forward)
# return ' '.join(destinations)
def get_forward_mailboxes(self):
for forward in self.forward.split():
if '@' not in forward:
try:
yield Mailbox.objects.get(name=forward)
except Mailbox.DoesNotExist:
pass
def get_mailboxes(self):
for mailbox in self.mailboxes.all():
yield mailbox
for mailbox in self.get_forward_mailboxes():
yield mailbox
class Autoresponse(models.Model):

12
orchestra/apps/mails/serializers.py
View file

@ -1,15 +1,23 @@
from django.forms import widgets
from django.utils.translation import ugettext, ugettext_lazy as _
from rest_framework import serializers
from orchestra.apps.accounts.serializers import AccountSerializerMixin
from orchestra.core.validators import validate_password
from .models import Mailbox, Address
class MailboxSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
password = serializers.CharField(max_length=128, label=_('Password'),
validators=[validate_password], write_only=True, required=False,
widget=widgets.PasswordInput)
class Meta:
model = Mailbox
# TODO 'use_custom_filtering',
fields = ('url', 'name', 'password', 'custom_filtering', 'addresses', 'is_active')
fields = (
'url', 'name', 'password', 'filtering', 'custom_filtering', 'addresses', 'is_active'
)
def validate_password(self, attrs, source):
""" POST only password """

49
orchestra/apps/mails/settings.py
View file

@ -1,4 +1,7 @@
import textwrap
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
MAILS_DOMAIN_MODEL = getattr(settings, 'MAILS_DOMAIN_MODEL', 'domains.Domain')
@ -14,23 +17,43 @@ MAILS_SIEVETEST_BIN_PATH = getattr(settings, 'MAILS_SIEVETEST_BIN_PATH',
'%(orchestra_root)s/bin/sieve-test')
MAILS_VIRTUSERTABLE_PATH = getattr(settings, 'MAILS_VIRTUSERTABLE_PATH',
'/etc/postfix/virtusertable')
MAILS_VIRTUAL_MAILBOX_MAPS_PATH = getattr(settings, 'MAILS_VIRTUAL_MAILBOX_MAPS_PATH',
'/etc/postfix/virtual_mailboxes')
MAILS_VIRTDOMAINS_PATH = getattr(settings, 'MAILS_VIRTDOMAINS_PATH',
'/etc/postfix/virtdomains')
MAILS_VIRTUAL_ALIAS_MAPS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_MAPS_PATH',
'/etc/postfix/virtual_aliases')
MAILS_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_DOMAINS_PATH',
'/etc/postfix/virtual_domains')
MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN = getattr(settings, 'MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN',
'orchestra.lan')
MAILS_PASSWD_PATH = getattr(settings, 'MAILS_PASSWD_PATH',
'/etc/dovecot/virtual_users')
'/etc/dovecot/passwd')
MAILS_DEFAUL_FILTERING = getattr(settings, 'MAILS_DEFAULT_FILTERING',
'require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];\n'
'\n'
'if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {\n'
' fileinto "Junk";\n'
' discard;\n'
'}'
)
MAILS_MAILBOX_FILTERINGS = getattr(settings, 'MAILS_MAILBOX_FILTERINGS', {
# value: (verbose_name, filter)
'DISABLE': (_("Disable"), ''),
'REJECT': (_("Reject spam"), textwrap.dedent("""
require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {
discard;
stop;
}""")),
'REDIRECT': (_("Archive spam"), textwrap.dedent("""
require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {
fileinto "Spam";
stop;
}""")),
'CUSTOM': (_("Custom filtering"), lambda mailbox: mailbox.custom_filtering),
})
MAILS_MAILBOX_DEFAULT_FILTERING = getattr(settings, 'MAILS_MAILBOX_DEFAULT_FILTERING', 'REDIRECT')

62
orchestra/apps/mails/tests/functional_tests/tests.py
View file

@ -4,8 +4,10 @@ import os
import poplib
import smtplib
import time
import textwrap
from email.mime.text import MIMEText
from django.apps import apps
from django.conf import settings as djsettings
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import CommandError
@ -33,8 +35,6 @@ class MailboxMixin(object):
def setUp(self):
super(MailboxMixin, self).setUp()
self.add_route()
# TODO fix this
from django.apps import apps
# clean resource relation from other tests
apps.get_app_config('resources').reload_relations()
djsettings.DEBUG = True
@ -92,7 +92,7 @@ class MailboxMixin(object):
def send_email(self, to, token):
msg = MIMEText(token)
msg['To'] = to
msg['From'] = 'orchestra@test.orchestra.lan'
msg['From'] = 'orchestra@%s' % self.MASTER_SERVER
msg['Subject'] = 'test'
server = smtplib.SMTP(self.MASTER_SERVER, 25)
try:
@ -176,7 +176,7 @@ class MailboxMixin(object):
password = '@!?%spppP001' % random_ascii(5)
self.add(username, password)
self.validate_mailbox(username)
self.addCleanup(self.delete, username)
# self.addCleanup(self.delete, username)
imap = self.login_imap(username, password)
self.disable(username)
self.assertRaises(imap.error, self.login_imap, username, password)
@ -212,6 +212,27 @@ class MailboxMixin(object):
self.send_email("%s@%s" % (name, domain), token)
self.validate_email(username, token)
def test_custom_filtering(self):
username = '%s_mailbox' % random_ascii(10)
password = '@!?%spppP001' % random_ascii(5)
folder = random_ascii(5)
filtering = textwrap.dedent("""
require "fileinto";
if true {
fileinto "%s";
stop;
}""" % folder)
self.add(username, password, filtering=filtering)
self.addCleanup(self.delete, username)
imap = self.login_imap(username, password)
imap.create(folder)
self.validate_mailbox(username)
token = random_ascii(100)
self.send_email("%s@%s" % (username, settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN), token)
home = Mailbox.objects.get(name=username).get_home()
sshrun(self.MASTER_SERVER,
"grep '%s' %s/Maildir/.%s/new/*" % (token, home, folder), display=False)
class RESTMailboxMixin(MailboxMixin):
def setUp(self):
@ -219,17 +240,22 @@ class RESTMailboxMixin(MailboxMixin):
self.rest_login()
@save_response_on_error
def add(self, username, password, quota=None):
def add(self, username, password, quota=None, filtering=None):
extra = {}
if quota:
extra = {
extra.update({
"resources": [
{
"name": "disk",
"allocated": quota
},
]
}
})
if filtering:
extra.update({
'filtering': 'CUSTOM',
'custom_filtering': filtering,
})
self.rest.mailboxes.create(name=username, password=password, **extra)
@save_response_on_error
@ -270,7 +296,7 @@ class AdminMailboxMixin(MailboxMixin):
self.admin_login()
@snapshot_on_error
def add(self, username, password, quota=None):
def add(self, username, password, quota=None, filtering=None):
url = self.live_server_url + reverse('admin:mails_mailbox_add')
self.selenium.get(url)
@ -285,17 +311,23 @@ class AdminMailboxMixin(MailboxMixin):
password_field.send_keys(password)
password_field = self.selenium.find_element_by_id('id_password2')
password_field.send_keys(password)
if quota is not None:
from orchestra.admin.utils import get_modeladmin
m = get_modeladmin(Mailbox)
print 't', type(m).inlines
print 'm', m.inlines
self.take_screenshot()
quota_field = self.selenium.find_element_by_id(
'id_resources-resourcedata-content_type-object_id-0-allocated')
quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
quota_field = self.selenium.find_element_by_id(quota_id)
quota_field.clear()
quota_field.send_keys(quota)
if filtering is not None:
filtering_input = self.selenium.find_element_by_id('id_filtering')
filtering_select = Select(filtering_input)
filtering_select.select_by_value("CUSTOM")
filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
filtering_inline.click()
time.sleep(0.5)
filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
filtering_field.send_keys(filtering)
name_field.submit()
self.assertNotEqual(url, self.selenium.current_url)

36
orchestra/apps/mails/validators.py
View file

@ -2,6 +2,7 @@ import hashlib
import os
import re
from django.core.management.base import CommandError
from django.core.validators import ValidationError, EmailValidator
from django.utils.translation import ugettext_lazy as _
@ -22,38 +23,33 @@ def validate_emailname(value):
raise ValidationError(msg)
#def validate_destination(value):
# """ space separated mailboxes or emails """
# for destination in value.split():
# msg = _("'%s' is not an existent mailbox" % destination)
# if '@' in destination:
# if not destination[-1].isalpha():
# raise ValidationError(msg)
# EmailValidator(destination)
# else:
# from .models import Mailbox
# if not Mailbox.objects.filter(user__username=destination).exists():
# raise ValidationError(msg)
# validate_emailname(destination)
def validate_forward(value):
""" space separated mailboxes or emails """
from .models import Mailbox
for destination in value.split():
EmailValidator(destination)
msg = _("'%s' is not an existent mailbox" % destination)
if '@' in destination:
if not destination[-1].isalpha():
raise ValidationError(msg)
EmailValidator(destination)
else:
if not Mailbox.objects.filter(user__username=destination).exists():
raise ValidationError(msg)
validate_emailname(destination)
def validate_sieve(value):
sieve_name = '%s.sieve' % hashlib.md5(value).hexdigest()
path = os.path.join(settings.EMAILS_SIEVETEST_PATH, sieve_name)
path = os.path.join(settings.MAILS_SIEVETEST_PATH, sieve_name)
with open(path, 'wb') as f:
f.write(value)
context = {
'orchestra_root': paths.get_orchestra_root()
}
sievetest = settings.EMAILS_SIEVETEST_BIN_PATH % context
test = run(' '.join([sievetest, path, '/dev/null']), display=False)
if test.return_code:
sievetest = settings.MAILS_SIEVETEST_BIN_PATH % context
try:
test = run(' '.join([sievetest, path, '/dev/null']), display=False)
except CommandError:
errors = []
for line in test.stderr.splitlines():
error = re.match(r'^.*(line\s+[0-9]+:.*)', line)

14
orchestra/apps/orchestration/admin.py
View file

@ -46,8 +46,8 @@ class RouteAdmin(admin.ModelAdmin):
class BackendOperationInline(admin.TabularInline):
model = BackendOperation
fields = ('action', 'instance_link')
readonly_fields = ('action', 'instance_link')
fields = ('action', 'content_object_link')
readonly_fields = ('action', 'content_object_link')
extra = 0
can_delete = False
@ -56,22 +56,22 @@ class BackendOperationInline(admin.TabularInline):
'all': ('orchestra/css/hide-inline-id.css',)
}
def instance_link(self, operation):
def content_object_link(self, operation):
try:
return admin_link('instance')(self, operation)
return admin_link('content_object')(self, operation)
except:
return _("deleted {0} {1}").format(
escape(operation.content_type), escape(operation.object_id)
)
instance_link.allow_tags = True
instance_link.short_description = _("Instance")
content_object_link.allow_tags = True
content_object_link.short_description = _("Content_object")
def has_add_permission(self, *args, **kwargs):
return False
def get_queryset(self, request):
queryset = super(BackendOperationInline, self).get_queryset(request)
return queryset.prefetch_related('instance')
return queryset.prefetch_related('content_object')
def display_mono(field):

6
orchestra/apps/orchestration/manager.py
View file

@ -14,9 +14,12 @@ logger = logging.getLogger(__name__)
def as_task(execute):
def wrapper(*args, **kwargs):
with db.transaction.commit_manually():
db.transaction.set_autocommit(False)
try:
log = execute(*args, **kwargs)
finally:
db.transaction.commit()
db.transaction.set_autocommit(True)
if log.state != log.SUCCESS:
send_report(execute, args, log)
return log
@ -25,7 +28,6 @@ def as_task(execute):
def close_connection(execute):
""" Threads have their own connection pool, closing it when finishing """
# TODO rewrite as context manager
def wrapper(*args, **kwargs):
log = execute(*args, **kwargs)
db.connection.close()

7
orchestra/apps/orchestration/middlewares.py
View file

@ -84,7 +84,12 @@ class OperationsMiddleware(object):
if not execute:
continue
instance = copy.copy(instance)
pending_operations.add(Operation.create(backend, instance, action))
operation = Operation.create(backend, instance, action)
if action != Operation.DELETE:
# usually we expect to be using last object state,
# except when we are deleting it
pending_operations.discard(operation)
pending_operations.add(operation)
def process_request(self, request):
""" Store request on a thread local variable """

12
orchestra/apps/orchestration/models.py
View file

@ -102,19 +102,23 @@ class BackendOperation(models.Model):
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
instance = generic.GenericForeignKey('content_type', 'object_id')
content_object = generic.GenericForeignKey('content_type', 'object_id')
class Meta:
verbose_name = _("Operation")
verbose_name_plural = _("Operations")
def __init__(self, *args, **kwargs):
self.instance = kwargs.pop('instance', None)
super(BackendOperation, self).__init__(*args, **kwargs)
def __unicode__(self):
return '%s.%s(%s)' % (self.backend, self.action, self.instance)
return '%s.%s(%s)' % (self.backend, self.action, self.instance or self.content_object)
def __hash__(self):
""" set() """
backend = getattr(self, 'backend', self.backend)
return hash(backend) + hash(self.instance) + hash(self.action)
return hash(backend) + hash(self.instance or self.content_object) + hash(self.action)
def __eq__(self, operation):
""" set() """
@ -122,7 +126,7 @@ class BackendOperation(models.Model):
@classmethod
def create(cls, backend, instance, action):
op = cls(backend=backend.get_name(), instance=instance, action=action)
op = cls(backend=backend.get_name(), instance=instance, content_object=instance, action=action)
op.backend = backend
return op

6
orchestra/apps/resources/admin.py
View file

@ -7,6 +7,7 @@ from orchestra.admin import ExtendedModelAdmin
from orchestra.admin.filters import UsedContentTypeFilter
from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
from orchestra.core import services
from orchestra.utils import database_ready
from .forms import ResourceForm
from .models import Resource, ResourceData, MonitorData
@ -135,7 +136,6 @@ def resource_inline_factory(resources):
return ResourceInline
from orchestra.utils import database_ready
def insert_resource_inlines():
# Clean previous state
for related in Resource._related:
@ -144,14 +144,12 @@ def insert_resource_inlines():
for inline in getattr(modeladmin_class, 'inlines', []):
if inline.__name__ == 'ResourceInline':
modeladmin_class.inlines.remove(inline)
modeladmin.inlines = modeladmin_class.inlines
for ct, resources in Resource.objects.group_by('content_type').iteritems():
inline = resource_inline_factory(resources)
model = ct.model_class()
modeladmin = get_modeladmin(model)
insertattr(model, 'inlines', inline)
modeladmin.inlines = type(modeladmin).inlines
if database_ready():
insert_resource_inlines()

7
orchestra/apps/resources/models.py
View file

@ -101,10 +101,9 @@ class Resource(models.Model):
elif task.crontab != self.crontab:
task.crontab = self.crontab
task.save(update_fields=['crontab'])
if created:
# This only work on tests because of multiprocessing used on real deployments
print 'saved'
apps.get_app_config('resources').reload_relations()
# This only work on tests (multiprocessing used on real deployments)
apps.get_app_config('resources').reload_relations()
# TODO touch wsgi.py for code reloading?
def delete(self, *args, **kwargs):
super(Resource, self).delete(*args, **kwargs)

5
orchestra/apps/services/models.py
View file

@ -230,10 +230,11 @@ class Service(models.Model):
def get_services(cls, instance):
cache = caches.get_request_cache()
ct = ContentType.objects.get_for_model(instance)
services = cache.get(ct)
key = 'services.Service-%i' % ct.pk
services = cache.get(key)
if services is None:
services = cls.objects.filter(content_type=ct, is_active=True)
cache.set(ct, services)
cache.set(key, services)
return services
# FIXME some times caching is nasty, do we really have to? make get_plugin more efficient?

6
orchestra/bin/orchestra-admin
View file

@ -130,8 +130,7 @@ function install_requirements () {
libxml2-dev \
libxslt1-dev \
wkhtmltopdf \
xvfb \
python-mysqldb"
xvfb"
PIP="django==1.7 \
django-celery-email==1.0.4 \
@ -159,7 +158,8 @@ function install_requirements () {
if $testing; then
APT="${APT} \
iceweasel \
dnsutils"
dnsutils \
python-mysqldb"
PIP="${PIP} \
selenium \
xvfbwrapper \

5
scripts/services/postfix.md
View file

@ -6,7 +6,7 @@
apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve
apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve dovecot-managesieved
echo 'mail_location = maildir:~/Maildir
mail_plugins = quota
@ -42,3 +42,6 @@ echo 'mailbox_transport = lmtp:unix:private/dovecot-lmtp' >> /etc/postfix/main.c
/etc/init.d/dovecot restart
/etc/init.d/postfix restart
# TODO check postfix and dovecot configs
# TODO crontab that deletes message +30 days on spam folders