Improved mailman backend

TODO.md

@@ -126,45 +126,21 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
     -# Required-Start:    $network $local_fs $remote_fs postgresql celeryd
     -# Required-Stop:     $network $local_fs $remote_fs postgresql celeryd
 
-
 * for list virtual_domains cleaning up we need to know the old domain name when a list changes its address domain, but this is not possible with the current design.
 * regenerate virtual_domains every time (configure a separate file for orchestra on postfix)
 * update_fields=[] doesn't trigger post save!
 
-* lists -> SaaS ?
-
-* move bill contact to bills apps
-
-
 * Backend optimization
     * fields = ()
     * ignore_fields = ()
     * based on a merge set of save(update_fields)
 
-
 * textwrap.dedent( \\)
-
-* accounts
-    * short name / long name, account name really needed? address? only minimal info..
-    * contact inlines
-    * autocreate stuff (email/<account>.orchestra.lan/plans)
-    * account username should be domain freiendly withot lines
-
-
 * parmiko write to a channel instead of transfering files?  http://sysadmin.circularvale.com/programming/paramiko-channel-hangs/
 
-* strip leading and trailing whitre spaces of most input fields
-
-* better modeling of the interdependency between webapps and websites (settings)
-* webapp options cfig agnostic
-
-* service.name / verbose_name instead of .description ?
-* miscellaneous.name / verbose_name
 
 * proforma without billing contact?
 
-* remove contact addresss, and use invoice contact for it (maybe move to contacts app again)
-
 * env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python manage.py test orchestra.apps.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest
 
 * Pangea modifications: domain registered/non-registered list_display and field with register link: inconsistent, what happen to related objects with a domain that is converted to register-only?
@@ -175,7 +151,7 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 
 * REST PERMISSIONS
 
-* caching based on def text2int(textnum, numwords={}):
+* caching based on def text2int(textnum, numwords={}) ?:
 
 
 * Subdomain saving should not trigger bind slave
@@ -184,3 +160,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 * prevent adding local email addresses on account.contacts account.email
 
 * Resource monitoring without ROUTE alert or explicit error
+
+
+* account.full_name account.short_name

orchestra/apps/accounts/admin.py

@@ -34,7 +34,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin)
             'fields': ('username', 'password1', 'password2',),
         }),
         (_("Personal info"), {
-            'fields': ('first_name', 'last_name', 'email', ('type', 'language'), 'comments'),
+            'fields': ('short_name', 'full_name', 'email', ('type', 'language'), 'comments'),
         }),
         (_("Permissions"), {
             'fields': ('is_superuser',)
@@ -45,7 +45,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin)
             'fields': ('username', 'password', 'main_systemuser_link')
         }),
         (_("Personal info"), {
-            'fields': ('first_name', 'last_name', 'email', ('type', 'language'), 'comments'),
+            'fields': ('short_name', 'full_name', 'email', ('type', 'language'), 'comments'),
         }),
         (_("Permissions"), {
             'fields': ('is_superuser', 'is_active')

orchestra/apps/accounts/migrations/0001_initial.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import django.utils.timezone
+import django.core.validators
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('systemusers', '__first__'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Account',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('password', models.CharField(max_length=128, verbose_name='password')),
+                ('last_login', models.DateTimeField(default=django.utils.timezone.now, verbose_name='last login')),
+                ('username', models.CharField(help_text='Required. 30 characters or fewer. Letters, digits and ./-/_ only.', unique=True, max_length=64, verbose_name='username', validators=[django.core.validators.RegexValidator(b'^[\\w.-]+$', 'Enter a valid username.', b'invalid')])),
+                ('first_name', models.CharField(max_length=30, verbose_name='first name', blank=True)),
+                ('last_name', models.CharField(max_length=30, verbose_name='last name', blank=True)),
+                ('email', models.EmailField(help_text='Used for password recovery', max_length=75, verbose_name='email address')),
+                ('type', models.CharField(default=b'INDIVIDUAL', max_length=32, verbose_name='type', choices=[(b'INDIVIDUAL', 'Individual'), (b'ASSOCIATION', 'Association'), (b'CUSTOMER', 'Customer'), (b'STAFF', 'Staff')])),
+                ('language', models.CharField(default=b'ca', max_length=2, verbose_name='language', choices=[(b'ca', 'Catalan'), (b'es', 'Spanish'), (b'en', 'English')])),
+                ('comments', models.TextField(max_length=256, verbose_name='comments', blank=True)),
+                ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),
+                ('is_active', models.BooleanField(default=True, help_text='Designates whether this account should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')),
+                ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')),
+                ('main_systemuser', models.ForeignKey(related_name='accounts_main', to='systemusers.SystemUser', null=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model,),
+        ),
+    ]

orchestra/apps/accounts/migrations/0002_auto_20141027_1414.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='account',
+            name='first_name',
+        ),
+        migrations.RemoveField(
+            model_name='account',
+            name='last_name',
+        ),
+        migrations.AddField(
+            model_name='account',
+            name='full_name',
+            field=models.CharField(default='', max_length=30, verbose_name='full name'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='account',
+            name='short_name',
+            field=models.CharField(default='', max_length=30, verbose_name='short name', blank=True),
+            preserve_default=False,
+        ),
+    ]

orchestra/apps/accounts/models.py

@@ -19,8 +19,8 @@ class Account(auth.AbstractBaseUser):
                         _("Enter a valid username."), 'invalid')])
     main_systemuser = models.ForeignKey(settings.ACCOUNTS_SYSTEMUSER_MODEL, null=True,
             related_name='accounts_main')
-    first_name = models.CharField(_("first name"), max_length=30, blank=True)
+    short_name = models.CharField(_("short name"), max_length=30, blank=True)
-    last_name = models.CharField(_("last name"), max_length=30, blank=True)
+    full_name = models.CharField(_("full name"), max_length=30)
     email = models.EmailField(_('email address'), help_text=_("Used for password recovery"))
     type = models.CharField(_("type"), choices=settings.ACCOUNTS_TYPES,
             max_length=32, default=settings.ACCOUNTS_DEFAULT_TYPE)
@@ -69,8 +69,8 @@ class Account(auth.AbstractBaseUser):
             self.save(update_fields=['main_systemuser'])
     
     def clean(self):
-        self.first_name = self.first_name.strip()
+        self.short_name = self.short_name.strip()
-        self.last_name = self.last_name.strip()
+        self.full_name = self.full_name.strip()
     
     def disable(self):
         self.is_active = False
@@ -93,12 +93,11 @@ class Account(auth.AbstractBaseUser):
         send_email_template(template, context, email_to, html=html, attachments=attachments)
     
     def get_full_name(self):
-        full_name = '%s %s' % (self.first_name, self.last_name)
+        return self.full_name or self.short_name or self.username
-        return full_name.strip() or self.username
     
     def get_short_name(self):
         """ Returns the short name for the user """
-        return self.first_name
+        return self.short_name or self.username or self.full_name
     
     def has_perm(self, perm, obj=None):
         """

orchestra/apps/accounts/serializers.py

@@ -7,7 +7,8 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Account
         fields = (
-            'url', 'username', 'type', 'language', 'date_joined', 'is_active'
+            'url', 'username', 'type', 'language', 'short_name', 'full_name', 'date_joined',
+            'is_active'
         )
 
 

orchestra/apps/bills/models.py

@@ -19,7 +19,8 @@ from . import settings
 class BillContact(models.Model):
     account = models.OneToOneField('accounts.Account', verbose_name=_("account"),
             related_name='billcontact')
-    name = models.CharField(_("name"), max_length=256)
+    name = models.CharField(_("name"), max_length=256, blank=True,
+            help_text=_("Account full name will be used when not provided"))
     address = models.TextField(_("address"))
     city = models.CharField(_("city"), max_length=128,
             default=settings.BILLS_CONTACT_DEFAULT_CITY)
@@ -31,6 +32,9 @@ class BillContact(models.Model):
     def __unicode__(self):
         return self.name
     
+    def get_name(self):
+        return self.name or self.account.get_full_name()
+
 
 class BillManager(models.Manager):
     def get_queryset(self):

orchestra/apps/bills/templates/bills/microspective-fee.html

@@ -87,7 +87,7 @@ hr {
 </div>
 
 <div id="buyer-details">
-    <span class="name">{{ buyer.name }}</span><br>
+    <span class="name">{{ buyer.get_name }}</span><br>
     {{ buyer.vat }}<br>
     {{ buyer.address }}<br>
     {{ buyer.zipcode }} - {{ buyer.city }}<br>

orchestra/apps/bills/templates/bills/microspective.html

@@ -23,7 +23,7 @@
 </div>
 <div id="seller-details">
     <div claas="address">
-        <span class="name">{{ seller.name }}</span>
+        <span class="name">{{ seller.get_name }}</span>
     </div>
     <div class="contact">
         <p>{{ seller.address }}<br>
@@ -58,7 +58,7 @@
     </div>
 </div>
 <div id="buyer-details">
-    <span class="name">{{ buyer.name }}</span><br>
+    <span class="name">{{ buyer.get_name }}</span><br>
     {{ buyer.vat }}<br>
     {{ buyer.address }}<br>
     {{ buyer.zipcode }} - {{ buyer.city }}<br>

orchestra/apps/lists/backends.py

@@ -43,7 +43,7 @@ class MailmanBackend(ServiceController):
     def exclude_virtual_alias_domain(self, context):
         address_domain = context['address_domain']
         if not List.objects.filter(address_domain=address_domain).exists():
-            self.append('sed -i "/^%(address_domain)s\s*/d" %(virtual_alias_domains)s' % context)
+            self.append('sed -i "/^%(address_domain)s\s*$/d" %(virtual_alias_domains)s' % context)
     
     def get_virtual_aliases(self, context):
         aliases = []
@@ -72,7 +72,7 @@ class MailmanBackend(ServiceController):
                     UPDATED_VIRTUAL_ALIAS=1
                 else
                     if [[ ! $(grep '^\s*%(address_name)s@%(address_domain)s\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then
-                        sed -i "s/^.*\s%(name)s\s*$//" %(virtual_alias)s
+                        sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s
                         echo '# %(banner)s\n%(aliases)s
                         ' >> %(virtual_alias)s
                         UPDATED_VIRTUAL_ALIAS=1
@@ -88,7 +88,7 @@ class MailmanBackend(ServiceController):
             # Cleanup shit
             self.append(textwrap.dedent("""\
                 if [[ ! $(grep '\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then
-                    sed -i "s/^.*\s%(name)s\s*$//" %(virtual_alias)s
+                    sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s
                 fi""" % context
             ))
         # Update
@@ -99,11 +99,11 @@ class MailmanBackend(ServiceController):
     def delete(self, mail_list):
         context = self.get_context(mail_list)
         self.exclude_virtual_alias_domain(context)
+        self.append('sed -i "/^\s*Generated by.*%(name)s\s*$/d" %(virtual_alias)s' % context)
         for address in self.addresses:
             context['address'] = address
-            self.append('sed -i "s/^.*\s%(name)s%(address)s\s*$//" %(virtual_alias)s' % context)
+            self.append('sed -i "/^.*\s%(name)s%(address)s\s*$/d" %(virtual_alias)s' % context)
-        # TODO remove
+        self.append("rmlist -a %(name)s" % context)
-        self.append("echo rmlist -a %(name)s" % context)
     
     def commit(self):
         context = self.get_context_files()
@@ -119,6 +119,10 @@ class MailmanBackend(ServiceController):
             'virtual_alias_domains': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH,
         }
     
+    def get_banner(self, mail_list):
+        banner = super(MailmanBackend, self).get_banner()
+        return '%s %s' % (banner, mail_list.name)
+    
     def get_context(self, mail_list):
         context = self.get_context_files()
         context.update({

orchestra/apps/orchestration/manager.py

@@ -30,10 +30,15 @@ def as_task(execute):
 def close_connection(execute):
     """ Threads have their own connection pool, closing it when finishing """
     def wrapper(*args, **kwargs):
-        log = execute(*args, **kwargs)
+        try:
-        db.connection.close()
+            log = execute(*args, **kwargs)
-        # Using the wrapper function as threader messenger for the execute output
+        except:
-        wrapper.log = log
+            raise
+        else:
+            # Using the wrapper function as threader messenger for the execute output
+            wrapper.log = log
+        finally:
+            db.connection.close()
     return wrapper
 
 

orchestra/apps/resources/serializers.py

@@ -8,10 +8,11 @@ from .models import Resource, ResourceData
 
 class ResourceSerializer(serializers.ModelSerializer):
     name = serializers.SerializerMethodField('get_name')
+    unit = serializers.Field()
     
     class Meta:
         model = ResourceData
-        fields = ('name', 'used', 'allocated')
+        fields = ('name', 'used', 'allocated', 'unit')
         read_only_fields = ('used',)
     
     def from_native(self, raw_data, files=None):

orchestra/apps/websites/backends/apache.py

@@ -167,7 +167,7 @@ class Apache2Backend(ServiceController):
             'group': site.get_groupname(),
             'sites_enabled': sites_enabled,
             'sites_available': "%s.conf" % os.path.join(sites_available, site.unique_name),
-            'logs': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
+            'logs': site.get_www_log_path(),
             'banner': self.get_banner(),
         }
         return context
@@ -237,7 +237,7 @@ class Apache2Traffic(ServiceMonitor):
     def get_context(self, site):
         last_date = self.get_last_date(site.pk)
         return {
-            'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
+            'log_file': '%s{,.1}' % site.get_www_log_path(),
             'last_date': last_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
             'object_id': site.pk,
         }

orchestra/apps/websites/models.py

@@ -56,6 +56,12 @@ class Website(models.Model):
     def get_groupname(self):
         return self.get_username()
     
+    def get_www_log_path(self):
+        context = {
+            'unique_name': self.unique_name
+        }
+        return settings.WEBSITES_WEBSITE_WWW_LOG_PATH % context
+
 
 class WebsiteOption(models.Model):
     website = models.ForeignKey(Website, verbose_name=_("web site"),

orchestra/apps/websites/settings.py

@@ -50,5 +50,5 @@ WEBSITES_WEBALIZER_PATH = getattr(settings, 'WEBSITES_WEBALIZER_PATH',
     '/home/httpd/webalizer/')
 
 
-WEBSITES_BASE_APACHE_LOGS = getattr(settings, 'WEBSITES_BASE_APACHE_LOGS',
+WEBSITES_WEBSITE_WWW_LOG_PATH = getattr(settings, 'WEBSITES_WEBSITE_WWW_LOG_PATH',
-    '/var/log/apache2/virtual/')
+    '/var/log/apache2/virtual/%(unique_name)s')