Improved resource monitoring

TODO.md

@@ -47,3 +47,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 
 * passlib; nano /usr/local/lib/python2.7/dist-packages/passlib/ext/django/utils.py SortedDict -> collections.OrderedDict
 * pip install pyinotify
+
+
+* Backend.operations dynamically generated based on defined methods

orchestra/admin/decorators.py

@@ -8,12 +8,13 @@ from django.utils.encoding import force_text
 
 
 def action_with_confirmation(action_name, extra_context={},
-        template='admin/controller/generic_confirmation.html'):
+                             template='admin/orchestra/generic_confirmation.html'):
     """ 
     Generic pattern for actions that needs confirmation step
     If custom template is provided the form must contain:
     <input type="hidden" name="post" value="generic_confirmation" />
     """
+
     def decorator(func, extra_context=extra_context, template=template):
         @wraps(func, assigned=available_attrs(func))
         def inner(modeladmin, request, queryset):

orchestra/apps/databases/backends.py

@@ -1,18 +1,25 @@
-from orchestra.apps.orchestration import ServiceBackend
+from django.utils.translation import ugettext_lazy as _
+
+from orchestra.apps.orchestration import ServiceController
+from orchestra.apps.resources import ServiceMonitor
 
 from . import settings
 
 
-class MySQLDBBackend(ServiceBackend):
+class MySQLDBBackend(ServiceController):
     verbose_name = "MySQL database"
     model = 'databases.Database'
     
     def save(self, database):
         if database.type == database.MYSQL:
             context = self.get_context(database)
-            self.append("mysql -e 'CREATE DATABASE `%(database)s`;'" % context)
+            self.append(
-            self.append("mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
+                "mysql -e 'CREATE DATABASE `%(database)s`;'" % context
-                        "  TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context)
+            )
+            self.append(
+                "mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
+                "  TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context
+            )
     
     def delete(self, database):
         if database.type == database.MYSQL:
@@ -30,21 +37,27 @@ class MySQLDBBackend(ServiceBackend):
         }
 
 
-class MySQLUserBackend(ServiceBackend):
+class MySQLUserBackend(ServiceController):
     verbose_name = "MySQL user"
     model = 'databases.DatabaseUser'
     
     def save(self, database):
         if database.type == database.MYSQL:
             context = self.get_context(database)
-            self.append("mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";'" % context)
+            self.append(
-            self.append("mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
+                "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";'" % context
-                        "  WHERE User=\"%(username)s\";'" % context)
+            )
+            self.append(
+                "mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
+                "  WHERE User=\"%(username)s\";'" % context
+            )
     
     def delete(self, database):
         if database.type == database.MYSQL:
             context = self.get_context(database)
-            self.append("mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context)
+            self.append(
+                "mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context
+            )
     
     def get_context(self, database):
         return {
@@ -54,7 +67,12 @@ class MySQLUserBackend(ServiceBackend):
         }
 
 
-class MySQLPermissionBackend(ServiceBackend):
+class MySQLPermissionBackend(ServiceController):
     model = 'databases.UserDatabaseRelation'
     verbose_name = "MySQL permission"
 
+
+class MysqlDisk(ServiceMonitor):
+    model = 'database.Database'
+    resource = ServiceMonitor.DISK
+    verbose_name = _("MySQL disk")

orchestra/apps/domains/backends.py

@@ -4,10 +4,10 @@ from django.utils.translation import ugettext_lazy as _
 
 from . import settings
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 
-class Bind9MasterDomainBackend(ServiceBackend):
+class Bind9MasterDomainBackend(ServiceController):
     verbose_name = _("Bind9 master domain")
     model = 'domains.Domain'
     related_models = (

orchestra/apps/lists/backends.py

@@ -1,11 +1,41 @@
 from django.template import Template, Context
+from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
+from orchestra.apps.resources import ServiceMonitor
 
 
-class MailmanBackend(ServiceBackend):
+class MailmanBackend(ServiceController):
     verbose_name = "Mailman"
     model = 'lists.List'
 
-    def save(self, mailinglist):
+
-        pass
+class MailmanTraffic(ServiceMonitor):
+    model = 'lists.List'
+    resource = ServiceMonitor.TRAFFIC
+    
+    def process(self, output):
+        for line in output.readlines():
+            listname, value = line.strip().slpit()
+    
+    def monitor(self, mailinglist):
+        self.append(
+            "LISTS=$(grep -v 'post to mailman' /var/log/mailman/post"
+            "  | grep size | cut -d'<' -f2 | cut -d'>' -f1 | sort | uniq"
+            "  | while read line; do \n"
+            "     grep \"$line\" post | head -n1 | awk {'print $8\" \"$11'}"
+            "       | sed 's/size=//' | sed 's/,//'\n"
+            "done)"
+        )
+        self.append(
+            'SUBS=""\n'
+            'while read LIST; do\n'
+            '   NAME=$(echo "$LIST" | awk {\'print $1\'})\n'
+            '   SIZE=$(echo "$LIST" | awk {\'print $2\'})\n'
+            '   if [[ ! $(echo -e "$SUBS" | grep "$NAME") ]]; then\n'
+            '       SUBS="${SUBS}${NAME} $(list_members "$NAME" | wc -l)\n"\n'
+            '   fi\n'
+            '   SUBSCRIBERS=$(echo -e "$SUBS" | grep "$NAME" | awk {\'print $2\'})\n'
+            '   echo "$NAME $(($SUBSCRIBERS*$SIZE))"\n'
+            'done <<< "$LISTS"'
+        )

orchestra/apps/orchestration/__init__.py

@@ -1 +1 @@
-from .backends import ServiceBackend
+from .backends import ServiceBackend, ServiceController

orchestra/apps/orchestration/admin.py

@@ -22,10 +22,11 @@ STATE_COLORS = {
 
 class RouteAdmin(admin.ModelAdmin):
     list_display = [
-        'id', 'backend', 'host', 'match', 'display_model', 'is_active'
+        'id', 'backend', 'host', 'match', 'display_model', 'display_actions',
+        'is_active'
     ]
     list_editable = ['backend', 'host', 'match', 'is_active']
-    list_filter = ['backend', 'host', 'is_active']
+    list_filter = ['host', 'is_active', 'backend']
     
     def display_model(self, route):
         try:
@@ -35,6 +36,14 @@ class RouteAdmin(admin.ModelAdmin):
     display_model.short_description = _("model")
     display_model.allow_tags = True
     
+    def display_actions(self, route):
+        try:
+            return '<br>'.join(route.get_backend().get_actions())
+        except KeyError:
+            return "<span style='color: red;'>NOT AVAILABLE</span>"
+    display_actions.short_description = _("actions")
+    display_actions.allow_tags = True
+
 
 class BackendOperationInline(admin.TabularInline):
     model = BackendOperation

orchestra/apps/orchestration/backends.py

@@ -23,6 +23,7 @@ class ServiceBackend(object):
     function_method = methods.Python
     type = 'task' # 'sync'
     ignore_fields = []
+    actions = []
     
     # TODO type: 'script', execution:'task'
     
@@ -37,6 +38,10 @@ class ServiceBackend(object):
     def __init__(self):
         self.cmds = []
     
+    @classmethod
+    def get_actions(cls):
+        return [ action for action in cls.actions if action in dir(cls) ]
+    
     @classmethod
     def get_name(cls):
         return cls.__name__
@@ -68,7 +73,7 @@ class ServiceBackend(object):
         choices = []
         for b in backends:
             # don't evaluate b.verbose_name ugettext_lazy
-            verbose = getattr(b.verbose_name, '_proxy____args', [None])
+            verbose = getattr(b.verbose_name, '_proxy____args', [b.verbose_name])
             if verbose[0]:
                 verbose = b.verbose_name
             else:
@@ -110,3 +115,12 @@ class ServiceBackend(object):
         the service once in bulk operations
         """
         pass
+
+
+class ServiceController(ServiceBackend):
+    actions = ('save', 'delete')
+    
+    @classmethod
+    def get_backends(cls):
+        """ filter controller classes """
+        return [ plugin for plugin in cls.plugins if ServiceController in plugin.__mro__ ]

orchestra/apps/orchestration/models.py

@@ -76,16 +76,14 @@ class BackendOperation(models.Model):
     """
     Encapsulates an operation, storing its related object, the action and the backend.
     """
-    SAVE = 'save'
     DELETE = 'delete'
-    ACTIONS = (
+    SAVE = 'save'
-        (SAVE, _("save")),
+    MONITOR = 'monitor'
-        (DELETE, _("delete")),
-    )
     
     log = models.ForeignKey('orchestration.BackendLog', related_name='operations')
+    # TODO backend and backend_class() (like content_type)
     backend_class = models.CharField(_("backend"), max_length=256)
-    action = models.CharField(_("action"), max_length=64, choices=ACTIONS)
+    action = models.CharField(_("action"), max_length=64)
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
     instance = generic.GenericForeignKey('content_type', 'object_id')
@@ -149,14 +147,21 @@ class Route(models.Model):
     
     @classmethod
     def get_servers(cls, operation):
-        backend_name = operation.backend.get_name()
+        # TODO use cached data sctructure and refactor
+        backend = operation.backend
+        servers = []
         try:
-            routes = cls.objects.filter(is_active=True, backend=backend_name)
+            routes = cls.objects.filter(is_active=True, backend=backend.get_name())
         except cls.DoesNotExist:
-            return []
+            return servers
-        safe_locals = { 'instance': operation.instance }
+        safe_locals = {
-        pks = [ route.pk for route in routes.all() if eval(route.match, safe_locals) ]
+            'instance': operation.instance
-        return [ route.host for route in routes.filter(pk__in=pks) ]
+        }
+        actions = backend.get_actions()
+        for route in routes:
+            if operation.action in actions and eval(route.match, safe_locals):
+                servers.append(route.host)
+        return servers
     
     def get_backend(self):
         for backend in ServiceBackend.get_backends():

orchestra/apps/resources/__init__.py

@@ -1 +1,4 @@
+from .backends import ServiceMonitor
+
+
 default_app_config = 'orchestra.apps.resources.apps.ResourcesConfig'

orchestra/apps/resources/admin.py

@@ -1,5 +1,3 @@
-import sys
-
 from django.contrib import admin
 from django.contrib.contenttypes import generic
 from django.utils.functional import cached_property
@@ -7,9 +5,10 @@ from django.utils.translation import ugettext_lazy as _
 
 from orchestra.admin.filters import UsedContentTypeFilter
 from orchestra.admin.utils import insertattr, get_modeladmin
+from orchestra.utils import running_syncdb
 
 from .forms import ResourceForm
-from .models import Resource, ResourceAllocation, Monitor, MonitorData
+from .models import Resource, ResourceData, MonitorData
 
 
 class ResourceAdmin(admin.ModelAdmin):
@@ -26,30 +25,24 @@ class ResourceAdmin(admin.ModelAdmin):
         resources = obj.content_type.resource_set.filter(is_active=True)
         inlines = []
         for inline in modeladmin.inlines:
-            if inline.model is ResourceAllocation:
+            if inline.model is ResourceData:
                 inline = resource_inline_factory(resources)
             inlines.append(inline)
         modeladmin.inlines = inlines
 
 
-class ResourceAllocationAdmin(admin.ModelAdmin):
+class ResourceDataAdmin(admin.ModelAdmin):
-    list_display = ('id', 'resource', 'content_object', 'value')
+    list_display = ('id', 'resource', 'used', 'allocated', 'last_update',) # TODO content_object
     list_filter = ('resource',)
 
 
-class MonitorAdmin(admin.ModelAdmin):
-    list_display = ('backend', 'resource', 'crontab')
-    list_filter = ('backend', 'resource')
-
-
 class MonitorDataAdmin(admin.ModelAdmin):
-    list_display = ('id', 'monitor', 'content_object', 'date', 'value')
+    list_display = ('id', 'monitor', 'date', 'value') # TODO content_object
     list_filter = ('monitor',)
 
 
 admin.site.register(Resource, ResourceAdmin)
-admin.site.register(ResourceAllocation, ResourceAllocationAdmin)
+admin.site.register(ResourceData, ResourceDataAdmin)
-admin.site.register(Monitor, MonitorAdmin)
 admin.site.register(MonitorData, MonitorDataAdmin)
 
 
@@ -68,7 +61,7 @@ def resource_inline_factory(resources):
             return forms
     
     class ResourceInline(generic.GenericTabularInline):
-        model = ResourceAllocation
+        model = ResourceData
         verbose_name_plural = _("resources")
         form = ResourceForm
         formset = ResourceInlineFormSet
@@ -84,7 +77,7 @@ def resource_inline_factory(resources):
     
     return ResourceInline
 
-if not 'migrate' in sys.argv and not 'syncdb' in sys.argv:
+if not running_syncdb():
     # not run during syncdb
     for resources in Resource.group_by_content_type():
         inline = resource_inline_factory(resources)

orchestra/apps/resources/apps.py

@@ -1,6 +1,8 @@
 from django.apps import AppConfig
 from django.contrib.contenttypes import generic
 
+from orchestra.utils import running_syncdb
+
 
 class ResourcesConfig(AppConfig):
     name = 'orchestra.apps.resources'
@@ -9,7 +11,8 @@ class ResourcesConfig(AppConfig):
     def ready(self):
         from .models import Resource
         # TODO execute on Resource.save()
-        relation = generic.GenericRelation('resources.ResourceAllocation')
+        if not running_syncdb():
-        for resources in Resource.group_by_content_type():
+            relation = generic.GenericRelation('resources.ResourceData')
-            model = resources[0].content_type.model_class()
+            for resources in Resource.group_by_content_type():
-            model.add_to_class('allocations', relation)
+                model = resources[0].content_type.model_class()
+                model.add_to_class('resources', relation)

orchestra/apps/resources/backends.py

@@ -0,0 +1,27 @@
+from orchestra.apps.orchestration import ServiceBackend
+
+
+class ServiceMonitor(ServiceBackend):
+    TRAFFIC = 'traffic'
+    DISK = 'disk'
+    MEMORY = 'memory'
+    CPU = 'cpu'
+    
+    actions = ('monitor', 'resource_exceeded', 'resource_recovery')
+    
+    @classmethod
+    def get_backends(cls):
+        """ filter monitor classes """
+        return [plugin for plugin in cls.plugins if ServiceMonitor in plugin.__mro__]
+    
+    def store(self, stdout):
+        """ object_id value """
+        for line in stdout.readlines():
+            line = line.strip()
+            object_id, value = line.split()
+            # TODO date
+            MonitorHistory.store(self.model, object_id, value, date)
+    
+    def execute(self, server):
+        log = super(MonitorBackend, self).execute(server)
+        return log

orchestra/apps/resources/forms.py

@@ -7,23 +7,33 @@ from orchestra.forms.widgets import ShowTextWidget, ReadOnlyWidget
 class ResourceForm(forms.ModelForm):
     verbose_name = forms.CharField(label=_("Name"), widget=ShowTextWidget(bold=True),
             required=False)
-    current = forms.CharField(label=_("Current"), widget=ShowTextWidget(),
+    used = forms.IntegerField(label=_("Used"), widget=ShowTextWidget(),
             required=False)
-    value = forms.CharField(label=_("Allocation"))
+    last_update = forms.CharField(label=_("Last update"), widget=ShowTextWidget(),
+            required=False)
+    allocated = forms.IntegerField(label=_("Allocated"))
     
     class Meta:
-        fields = ('verbose_name', 'current', 'value',)
+        fields = ('verbose_name', 'used', 'last_update', 'allocated',)
     
     def __init__(self, *args, **kwargs):
         self.resource = kwargs.pop('resource', None)
         super(ResourceForm, self).__init__(*args, **kwargs)
         if self.resource:
             self.fields['verbose_name'].initial = self.resource.verbose_name
-            self.fields['current'].initial = self.resource.get_current()
+            self.fields['used'].initial = self.resource.get_current()
             if self.resource.ondemand:
-                self.fields['value'].widget = ReadOnlyWidget('')
+                self.fields['allocated'].required = False
+                self.fields['allocated'].widget = ReadOnlyWidget(None, '')
             else:
-                self.fields['value'].initial = self.resource.default_allocation
+                self.fields['allocated'].required = True
+                self.fields['allocated'].initial = self.resource.default_allocation
+    
+    def has_changed(self):
+        """ Make sure resourcedata objects are created for all resources """
+        if not self.instance.pk:
+            return True
+        return super(ResourceForm, self).has_changed()
     
     def save(self, *args, **kwargs):
         self.instance.resource_id = self.resource.pk

orchestra/apps/resources/models.py

@@ -7,13 +7,25 @@ from django.core import validators
 from django.utils.translation import ugettext_lazy as _
 from djcelery.models import PeriodicTask, CrontabSchedule
 
+from orchestra.models.fields import MultiSelectField
 from orchestra.utils.apps import autodiscover
 
+from .backends import ServiceMonitor
+
 
 class Resource(models.Model):
-    MONTHLY = 'MONTHLY'
+    """
+    Defines a resource, a resource is basically an interpretation of data
+    gathered by a Monitor
+    """
+    
+    LAST = 'LAST'
+    MONTHLY_SUM = 'MONTHLY_SUM'
+    MONTHLY_AVG = 'MONTHLY_AVG'
     PERIODS = (
-        (MONTHLY, _('Monthly')),
+        (LAST, _("Last")),
+        (MONTHLY_SUM, _("Monthly Sum")),
+        (MONTHLY_AVG, _("Monthly Average")),
     )
     
     name = models.CharField(_("name"), max_length=32, unique=True,
@@ -24,11 +36,14 @@ class Resource(models.Model):
     verbose_name = models.CharField(_("verbose name"), max_length=256, unique=True)
     content_type = models.ForeignKey(ContentType) # TODO filter by servicE?
     period = models.CharField(_("period"), max_length=16, choices=PERIODS,
-            default=MONTHLY)
+            default=LAST)
-    ondemand = models.BooleanField(default=False)
+    ondemand = models.BooleanField(_("on demand"), default=False)
-    default_allocation = models.PositiveIntegerField(null=True, blank=True)
+    default_allocation = models.PositiveIntegerField(_("default allocation"),
-    is_active = models.BooleanField(default=True)
+            null=True, blank=True)
-    disable_trigger = models.BooleanField(default=False)
+    is_active = models.BooleanField(_("is active"), default=True)
+    disable_trigger = models.BooleanField(_("disable trigger"), default=False)
+    monitors = MultiSelectField(_("monitors"), max_length=256,
+            choices=ServiceMonitor.get_choices())
     
     def __unicode__(self):
         return self.name
@@ -53,47 +68,58 @@ class Resource(models.Model):
         today = datetime.date.today()
         result = 0
         has_result = False
-        for monitor in self.monitors.all():
+        for monitor in self.monitors:
-            has_result = True
+            dataset = MonitorData.objects.filter(monitor=monitor)
-            if self.period == self.MONTHLY:
+            if self.period == self.MONTHLY_AVG:
-                data = monitor.dataset.filter(date__year=today.year,
+                try:
-                                              date__month=today.month)
+                    last = dataset.latest()
-                result += data.aggregate(models.Sum('value'))['value__sum']
+                except MonitorData.DoesNotExist:
+                    continue
+                has_result = True
+                epoch = datetime(year=today.year, month=today.month, day=1)
+                total = (epoch-last.date).total_seconds()
+                dataset = dataset.filter(date__year=today.year,
+                                                 date__month=today.month)
+                for data in dataset:
+                    slot = (previous-data.date).total_seconds()
+                    result += data.value * slot/total
+            elif self.period == self.MONTHLY_SUM:
+                data = dataset.filter(date__year=today.year,
+                                      date__month=today.month)
+                value = data.aggregate(models.Sum('value'))['value__sum']
+                if value:
+                    has_result = True
+                    result += value
+            elif self.period == self.LAST:
+                try:
+                    result += dataset.latest().value
+                except MonitorData.DoesNotExist:
+                    continue
+                has_result = True
             else:
                 raise NotImplementedError("%s support not implemented" % self.period)
         return result if has_result else None
 
 
-class ResourceAllocation(models.Model):
+class ResourceData(models.Model):
+    """ Stores computed resource usage and allocation """
     resource = models.ForeignKey(Resource)
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
-    value = models.PositiveIntegerField()
+    used = models.PositiveIntegerField(null=True)
+    last_update = models.DateTimeField(null=True)
+    allocated = models.PositiveIntegerField(null=True)
     
     content_object = generic.GenericForeignKey()
     
     class Meta:
         unique_together = ('resource', 'content_type', 'object_id')
-
+        verbose_name_plural = _("resource data")
-
-autodiscover('monitors')
-
-
-class Monitor(models.Model):
-    backend = models.CharField(_("backend"), max_length=256,)
-#            choices=MonitorBackend.get_choices())
-    resource = models.ForeignKey(Resource, related_name='monitors')
-    crontab = models.ForeignKey(CrontabSchedule)
-    
-    class Meta:
-        unique_together=('backend', 'resource')
-    
-    def __unicode__(self):
-        return self.backend
-
 
 class MonitorData(models.Model):
-    monitor = models.ForeignKey(Monitor, related_name='dataset')
+    """ Stores monitored data """
+    monitor = models.CharField(_("monitor"), max_length=256,
+            choices=ServiceMonitor.get_choices())
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
     date = models.DateTimeField(auto_now_add=True)
@@ -101,5 +127,9 @@ class MonitorData(models.Model):
     
     content_object = generic.GenericForeignKey()
     
+    class Meta:
+        get_latest_by = 'date'
+        verbose_name_plural = _("monitor data")
+    
     def __unicode__(self):
         return str(self.monitor)

orchestra/apps/resources/serializers.py

@@ -1,27 +1,24 @@
 from rest_framework import serializers
 
 from orchestra.api import router
+from orchestra.utils import running_syncdb
 
-from .models import Resource, ResourceAllocation
+from .models import Resource, ResourceData
 
 
 class ResourceSerializer(serializers.ModelSerializer):
     name = serializers.SerializerMethodField('get_name')
-    current = serializers.SerializerMethodField('get_current')
-    allocation = serializers.IntegerField(source='value')
     
     class Meta:
-        model = ResourceAllocation
+        model = ResourceData
-        fields = ('name', 'current', 'allocation')
+        fields = ('name', 'used', 'allocated')
+        read_only_fields = ('used',)
     
     def get_name(self, instance):
         return instance.resource.name
 
-    def get_current(self, instance):
-        return instance.resource.get_current()
 
-
+if not running_syncdb():
-for resources in Resource.group_by_content_type():
+    for resources in Resource.group_by_content_type():
-    model = resources[0].content_type.model_class()
+        model = resources[0].content_type.model_class()
-    router.insert(model, 'resources', ResourceSerializer, required=False,
+        router.insert(model, 'resources', ResourceSerializer, required=False)
-                  source='allocations')

orchestra/apps/resources/tasks.py

@@ -0,0 +1,14 @@
+from celery import shared_task
+
+from .backends import ServiceMonitor
+
+
+@shared_task
+def monitor(backend_name):
+    routes = Route.objects.filter(is_active=True, backend=backend_name)
+    for route in routes:
+        pass
+    for backend in ServiceMonitor.get_backends():
+        if backend.get_name() == backend_name:
+    # TODO execute monitor BackendOperation
+           pass

orchestra/apps/users/admin.py

@@ -15,7 +15,7 @@ from .roles.filters import role_list_filter_factory
 
 
 class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
-    list_display = ('username', 'is_main')
+    list_display = ('username', 'display_is_main')
     list_filter = ('is_staff', 'is_superuser', 'is_active')
     fieldsets = (
         (None, {
@@ -25,7 +25,7 @@ class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
             'fields': ('first_name', 'last_name', 'email')
         }),
         (_("Permissions"), {
-            'fields': ('is_active', 'is_staff', 'is_superuser', 'is_admin', 'is_main')
+            'fields': ('is_active', 'is_staff', 'is_superuser', 'display_is_main')
         }),
         (_("Important dates"), {
             'fields': ('last_login', 'date_joined')
@@ -38,7 +38,7 @@ class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
         }),
     )
     search_fields = ['username', 'account__user__username']
-    readonly_fields = ('is_main', 'account_link')
+    readonly_fields = ('display_is_main', 'account_link')
     change_readonly_fields = ('username',)
     filter_horizontal = ()
     add_form = UserCreationForm
@@ -46,10 +46,10 @@ class UserAdmin(AccountAdminMixin, auth.UserAdmin, ExtendedModelAdmin):
     roles = []
     ordering = ('-id',)
     
-    
+    def display_is_main(self, instance):
-    def is_main(self, user):
+        return instance.is_main
-        return user.account.user == user
+    display_is_main.short_description = _("is main")
-    is_main.boolean = True
+    display_is_main.boolean = True
     
     def get_urls(self):
         """ Returns the additional urls for the change view links """

orchestra/apps/users/backends.py

@@ -1,11 +1,12 @@
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
+from orchestra.apps.resources import ServiceMonitor
 
 from . import settings
 
 
-class SystemUserBackend(ServiceBackend):
+class SystemUserBackend(ServiceController):
     verbose_name = _("System User")
     model = 'users.User'
     ignore_fields = ['last_login']
@@ -39,3 +40,22 @@ class SystemUserBackend(ServiceBackend):
         }
         context['home'] = settings.USERS_SYSTEMUSER_HOME % context
         return context
+
+
+class SystemUserDisk(ServiceMonitor):
+    model = 'users.User'
+    resource = ServiceMonitor.DISK
+    verbose_name = _('System user disk')
+    
+    def monitor(self, user):
+        context = self.get_context(user)
+        self.append("du -s %(home)s | {\n"
+                    "   read value\n"
+                    "   echo '%(username)s' $value\n"
+                    "}" % context)
+    
+    def process(self, output):
+        # TODO transaction
+        for line in output.readlines():
+            username, value = line.strip().slpit()
+            History.store(object_id=user_id, value=value)

orchestra/apps/users/models.py

@@ -36,6 +36,10 @@ class User(auth.AbstractBaseUser):
     USERNAME_FIELD = 'username'
     REQUIRED_FIELDS = ['email']
     
+    @property
+    def is_main(self):
+        return self.account.user == self
+    
     def get_full_name(self):
         full_name = '%s %s' % (self.first_name, self.last_name)
         return full_name.strip() or self.username

orchestra/apps/users/roles/mail/backends.py

@@ -3,12 +3,13 @@ import os
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
+from orchestra.apps.resources import ServiceMonitor
 
 from . import settings
 
 
-class MailSystemUserBackend(ServiceBackend):
+class MailSystemUserBackend(ServiceController):
     verbose_name = _("Mail system user")
     model = 'mail.Mailbox'
     
@@ -62,7 +63,7 @@ class MailSystemUserBackend(ServiceBackend):
         return context
 
 
-class PostfixAddressBackend(ServiceBackend):
+class PostfixAddressBackend(ServiceController):
     verbose_name = _("Postfix address")
     model = 'mail.Address'
     
@@ -132,12 +133,12 @@ class PostfixAddressBackend(ServiceBackend):
         return context
 
 
-class AutoresponseBackend(ServiceBackend):
+class AutoresponseBackend(ServiceController):
     verbose_name = _("Mail autoresponse")
     model = 'mail.Autoresponse'
     
-    def save(self, autoresponse):
-        pass
 
-    def delete(self, autoresponse):
+class MailDisk(ServiceMonitor):
-        pass
+    model = 'email.Mailbox'
+    resource = ServiceMonitor.DISK
+    verbose_name = _("Mail disk")

orchestra/apps/vps/backends.py

@@ -0,0 +1,20 @@
+from django.utils.translation import ugettext_lazy as _
+
+from orchestra.apps.resources import ServiceMonitor
+
+
+class OpenVZDisk(ServiceMonitor):
+    model = 'vps.VPS'
+    resource = ServiceMonitor.DISK
+
+
+class OpenVZMemory(ServiceMonitor):
+    model = 'vps.VPS'
+    resource = ServiceMonitor.MEMORY
+
+
+class OpenVZTraffic(ServiceMonitor):
+    model = 'vps.VPS'
+    resource = ServiceMonitor.TRAFFIC
+
+

orchestra/apps/webapps/backends/awstats.py

@@ -1,12 +1,9 @@
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from . import WebAppServiceMixin
 
 
-class AwstatsBackend(WebAppServiceMixin, ServiceBackend):
+class AwstatsBackend(WebAppServiceMixin, ServiceController):
     verbose_name = _("Awstats")
-    
-    def save(self, webapp):
-        pass

orchestra/apps/webapps/backends/dokuwikimu.py

@@ -1,11 +1,12 @@
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from . import WebAppServiceMixin
 from .. import settings
 
-class DokuWikiMuBackend(WebAppServiceMixin, ServiceBackend):
+
+class DokuWikiMuBackend(WebAppServiceMixin, ServiceController):
     verbose_name = _("DokuWiki multisite")
     
     def save(self, webapp):

orchestra/apps/webapps/backends/drupalmu.py

@@ -2,13 +2,13 @@ import os
 
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from . import WebAppServiceMixin
 from .. import settings
 
 
-class DrupalMuBackend(WebAppServiceMixin, ServiceBackend):
+class DrupalMuBackend(WebAppServiceMixin, ServiceController):
     verbose_name = _("Drupal multisite")
     
     def save(self, webapp):

orchestra/apps/webapps/backends/phpfcgid.py

@@ -2,13 +2,13 @@ import os
 
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from . import WebAppServiceMixin
 from .. import settings
 
 
-class PHPFcgidBackend(WebAppServiceMixin, ServiceBackend):
+class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
     verbose_name = _("PHP-Fcgid")
     
     def save(self, webapp):

orchestra/apps/webapps/backends/phpfpm.py

@@ -3,13 +3,13 @@ import os
 from django.template import Template, Context
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from . import WebAppServiceMixin
 from .. import settings
 
 
-class PHPFPMBackend(WebAppServiceMixin, ServiceBackend):
+class PHPFPMBackend(WebAppServiceMixin, ServiceController):
     verbose_name = _("PHP-FPM")
     
     def save(self, webapp):

orchestra/apps/webapps/backends/static.py

@@ -1,11 +1,11 @@
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from . import WebAppServiceMixin
 
 
-class StaticBackend(WebAppServiceMixin, ServiceBackend):
+class StaticBackend(WebAppServiceMixin, ServiceController):
     verbose_name = _("Static")
     
     def save(self, webapp):

orchestra/apps/webapps/backends/wordpressmu.py

@@ -4,13 +4,13 @@ import sys
 import requests
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from . import WebAppServiceMixin
 from .. import settings
 
 
-class WordpressMuBackend(WebAppServiceMixin, ServiceBackend):
+class WordpressMuBackend(WebAppServiceMixin, ServiceController):
     verbose_name = _("Wordpress multisite")
     
     @property

orchestra/apps/websites/backends/apache.py

@@ -3,12 +3,13 @@ import os
 from django.template import Template, Context
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
+from orchestra.apps.resources import ServiceMonitor
 
 from .. import settings
 
 
-class Apache2Backend(ServiceBackend):
+class Apache2Backend(ServiceController):
     model = 'websites.Website'
     related_models = (('websites.Content', 'website'),)
     verbose_name = _("Apache 2")
@@ -173,3 +174,58 @@ class Apache2Backend(ServiceBackend):
             'fpm_port': content.webapp.get_fpm_port(),
         })
         return context
+
+
+class Apache2Traffic(ServiceMonitor):
+    model = 'websites.Website'
+    resource = ServiceMonitor.TRAFFIC
+    verbose_name = _("Apache 2 Traffic")
+    
+    def monitor(self, site):
+        context = self.get_context(site)
+        self.append("""
+            awk 'BEGIN {
+                ini = "%(start_date)s";
+                end = "%(end_date)s";
+                
+                months["Jan"]="01";
+                months["Feb"]="02";
+                months["Mar"]="03";
+                months["Apr"]="04";
+                months["May"]="05";
+                months["Jun"]="06";
+                months["Jul"]="07";
+                months["Aug"]="08";
+                months["Sep"]="09";
+                months["Oct"]="10";
+                months["Nov"]="11";
+                months["Dec"]="12";
+            } {
+                date = substr($4,2)
+                year = substr(date,8,4)
+                month = months[substr(date,4,3)];
+                day = substr(date,1,2)
+                hour = substr(date,13,2)
+                minute = substr(date,16,2)
+                second = substr(date,19,2);
+                line_date = year month day hour minute second
+                if ( line_date > ini && line_date < end)
+                    if ( $10 == "" )
+                        sum+=$9
+                    else
+                        sum+=$10;
+            } END {
+                print sum;
+            }' %(log_file)s | {
+                read value
+                echo %(site_name)s $value
+            }
+            """ % context)
+    
+    def get_context(self, site):
+        return {
+            'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
+            'start_date': '',
+            'end_date': '',
+            'site_name': '',
+        }

orchestra/apps/websites/backends/webalizer.py

@@ -3,12 +3,12 @@ from functools import partial
 
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.apps.orchestration import ServiceBackend
+from orchestra.apps.orchestration import ServiceController
 
 from .. import settings
 
 
-class WebalizerBackend(ServiceBackend):
+class WebalizerBackend(ServiceController):
     verbose_name = _("Webalizer")
     model = 'websites.Content'
     

orchestra/apps/websites/resources.py

@@ -1,291 +0,0 @@
-from . import settings
-
-
-class ServiceBackend(object):
-    """
-    Service management backend base class
-    
-    It uses the _unit of work_ design principle, which allows bulk operations to
-    be conviniently supported. Each backend generates the configuration for all
-    the changes of all modified objects, reloading the daemon just once.
-    """
-    verbose_name = None
-    model = None
-    related_models = () # ((model, accessor__attribute),)
-    script_method = methods.BashSSH
-    function_method = methods.Python
-    type = 'task' # 'sync'
-    ignore_fields = []
-    
-    # TODO type: 'script', execution:'task'
-    
-    __metaclass__ = plugins.PluginMount
-    
-    def __unicode__(self):
-        return type(self).__name__
-    
-    def __str__(self):
-        return unicode(self)
-    
-    def __init__(self):
-        self.cmds = []
-    
-    @classmethod
-    def get_name(cls):
-        return cls.__name__
-    
-    @classmethod
-    def is_main(cls, obj):
-        opts = obj._meta
-        return cls.model == '%s.%s' % (opts.app_label, opts.object_name)
-    
-    @classmethod
-    def get_related(cls, obj):
-        opts = obj._meta
-        model = '%s.%s' % (opts.app_label, opts.object_name)
-        for rel_model, field in cls.related_models:
-            if rel_model == model:
-                related = obj
-                for attribute in field.split('__'):
-                    related = getattr(related, attribute)
-                return related
-        return None
-    
-    @classmethod
-    def get_backends(cls):
-        return cls.plugins
-    
-    @classmethod
-    def get_choices(cls):
-        backends = cls.get_backends()
-        choices = ( (b.get_name(), b.verbose_name or b.get_name()) for b in backends )
-        return sorted(choices, key=lambda e: e[1])
-    
-    def get_banner(self):
-        time = datetime.now().strftime("%h %d, %Y %I:%M:%S")
-        return "Generated by Orchestra %s" % time
-    
-    def append(self, *cmd):
-        # aggregate commands acording to its execution method
-        if isinstance(cmd[0], basestring):
-            method = self.script_method
-            cmd = cmd[0]
-        else:
-            method = self.function_method
-            cmd = partial(*cmd)
-        if not self.cmds or self.cmds[-1][0] != method:
-            self.cmds.append((method, [cmd]))
-        else:
-            self.cmds[-1][1].append(cmd)
-    
-    def execute(self, server):
-        from .models import BackendLog
-        state = BackendLog.STARTED if self.cmds else BackendLog.SUCCESS
-        log = BackendLog.objects.create(backend=self.get_name(), state=state, server=server)
-        for method, cmds in self.cmds:
-            method(log, server, cmds)
-            if log.state != BackendLog.SUCCESS:
-                break
-        return log
-
-
-def ServiceController(ServiceBackend):
-    def save(self, obj)
-        raise NotImplementedError
-    
-    def delete(self, obj):
-        raise NotImplementedError
-    
-    def commit(self):
-        """
-        apply the configuration, usually reloading a service 
-        reloading a service is done in a separated method in order to reload 
-        the service once in bulk operations
-        """
-        pass
-
-
-class ServiceMonitor(ServiceBackend):
-    TRAFFIC = 'traffic'
-    DISK = 'disk'
-    MEMORY = 'memory'
-    CPU = 'cpu'
-    
-    def prepare(self):
-        pass
-    
-    def store(self, stdout):
-        """ object_id value """
-        for line in stdout.readlines():
-            line = line.strip()
-            object_id, value = line.split()
-            # TODO date
-            MonitorHistory.store(self.model, object_id, value, date)
-
-    def monitor(self, obj):
-        raise NotImplementedError
-    
-    def trigger(self, obj):
-        raise NotImplementedError
-    
-    def execute(self, server):
-        log = super(MonitorBackend, self).execute(server)
-        
-        return log
-
-
-class AccountDisk(MonitorBackend):
-    model = 'accounts.Account'
-    resource = MonitorBackend.DISK
-    verbose_name = 'Disk'
-    
-    def monitor(self, user):
-        context = self.get_context(user)
-        self.append("du -s %(home)s | {\n"
-                    "   read value\n"
-                    "   echo '%(username)s' $value\n"
-                    "}" % context)
-    
-     def process(self, output):
-        # TODO transaction
-        for line in output.readlines():
-            username, value = line.strip().slpit()
-            History.store(object_id=user_id, value=value)
-
-
-class MailmanTraffic(MonitorBackend):
-    model = 'lists.List'
-    resource = MonitorBackend.TRAFFIC
-    
-    def process(self, output):
-        for line in output.readlines():
-            listname, value = line.strip().slpit()
-    
-    def monitor(self, mailinglist):
-        self.append("LISTS=$(grep -v 'post to mailman' /var/log/mailman/post"
-                    "  | grep size | cut -d'<' -f2 | cut -d'>' -f1 | sort | uniq"
-                    "  | while read line; do \n"
-                    "     grep \"$line\" post | head -n1 | awk {'print $8\" \"$11'}"
-                    "       | sed 's/size=//' | sed 's/,//'\n"
-                    "done)")
-        self.append('SUBS=""\n'
-                    'while read LIST; do\n'
-                    '   NAME=$(echo "$LIST" | awk {\'print $1\'})\n'
-                    '   SIZE=$(echo "$LIST" | awk {\'print $2\'})\n'
-                    '   if [[ ! $(echo -e "$SUBS" | grep "$NAME") ]]; then\n'
-                    '       SUBS="${SUBS}${NAME} $(list_members "$NAME" | wc -l)\n"\n'
-                    '   fi\n'
-                    '   SUBSCRIBERS=$(echo -e "$SUBS" | grep "$NAME" | awk {\'print $2\'})\n'
-                    '   echo "$NAME $(($SUBSCRIBERS*$SIZE))"\n'
-                    'done <<< "$LISTS"')
-
-
-class MailDisk(MonitorBackend):
-    model = 'email.Mailbox'
-    resource = MonitorBackend.DISK
-    verbose_name = _("Mail disk")
-    
-    def process(self, output):
-        pass
-    
-    def monitor(self, mail):
-        pass
-
-
-class MysqlDisk(MonitorBackend):
-    model = 'database.Database'
-    resource = MonitorBackend.DISK
-    verbose_name = _("MySQL disk")
-    
-    def process(self, output):
-        pass
-    
-    def monitor(self, db):
-        pass
-
-
-class OpenVZDisk(MonitorBackend):
-    model = 'vps.VPS'
-    resource = MonitorBackend.DISK
-
-
-class OpenVZMemory(MonitorBackend):
-    model = 'vps.VPS'
-    resource = MonitorBackend.MEMORY
-
-
-class OpenVZTraffic(MonitorBackend):
-    model = 'vps.VPS'
-    resource = MonitorBackend.TRAFFIC
-
-
-class Apache2Traffic(MonitorBackend):
-    model = 'websites.Website'
-    resource = MonitorBackend.TRAFFIC
-    verbose_name = _("Apache2 Traffic")
-    
-    def monitor(self, site):
-        context = self.get_context(site)
-        self.append("""
-            awk 'BEGIN {
-                ini = "%(start_date)s";
-                end = "%(end_date)s";
-                
-                months["Jan"]="01";
-                months["Feb"]="02";
-                months["Mar"]="03";
-                months["Apr"]="04";
-                months["May"]="05";
-                months["Jun"]="06";
-                months["Jul"]="07";
-                months["Aug"]="08";
-                months["Sep"]="09";
-                months["Oct"]="10";
-                months["Nov"]="11";
-                months["Dec"]="12";
-            } {
-                date = substr($4,2)
-                year = substr(date,8,4)
-                month = months[substr(date,4,3)];
-                day = substr(date,1,2)
-                hour = substr(date,13,2)
-                minute = substr(date,16,2)
-                second = substr(date,19,2);
-                line_date = year month day hour minute second
-                if ( line_date > ini && line_date < end)
-                    if ( $10 == "" )
-                        sum+=$9
-                    else
-                        sum+=$10;
-            } END {
-                print sum;
-            }' %(log_file)s | {
-                read value
-                echo %(site_name)s $value
-            }
-            """ % context)
-    
-    def trigger(self, site):
-        pass
-    
-    def get_context(self, site):
-        return {
-            'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name)
-            
-        }
-
-# start_date and end_date expected format: YYYYMMDDhhmmss
-
-function get_traffic(){
-
-
-RESULT=$(get_traffic)
-
-if [[ $RESULT ]]; then 
-    echo $RESULT
-else
-    echo 0
-fi
-
-return 0
-

orchestra/forms/widgets.py

@@ -4,6 +4,11 @@ from django.utils.encoding import force_text
 
 
 class ShowTextWidget(forms.Widget):
+    def __init__(self, *args, **kwargs):
+        for kwarg in ['bold', 'warning', 'hidden']:
+            setattr(self, kwarg, kwargs.pop(kwarg, False))
+        super(ShowTextWidget, self).__init__(*args, **kwargs)
+    
     def render(self, name, value, attrs):
         value = force_text(value)
         if value is None:
@@ -20,11 +25,6 @@ class ShowTextWidget(forms.Widget):
             final_value = u'%s<input type="hidden" name="%s" value="%s"/>' % (final_value, name, value)
         return mark_safe(final_value)
     
-    def __init__(self, *args, **kwargs):
-        for kwarg in ['bold', 'warning', 'hidden']:
-            setattr(self, kwarg, kwargs.pop(kwarg, False))
-        super(ShowTextWidget, self).__init__(*args, **kwargs)
-        
     def _has_changed(self, initial, data):
         return False
 

orchestra/models/fields.py

@@ -54,4 +54,4 @@ class MultiSelectField(models.CharField):
 
 if isinstalled('south'):
     from south.modelsinspector import add_introspection_rules
-    add_introspection_rules([], ["^controller\.models\.fields\.MultiSelectField"])
+    add_introspection_rules([], ["^orchestra\.models\.fields\.MultiSelectField"])

orchestra/static/orchestra/css/adminextraprettystyle.css

@@ -5,18 +5,18 @@ body {
 
 #header #branding h1 {
     margin: 0;
-    padding: 5px 10px;
+    padding: 2px 10px;
-    background: transparent url(/static/orchestra/images/orchestra-logo.png) 10px 5px no-repeat;
+    background: transparent url(/static/orchestra/images/orchestra-logo.png) 10px 2px no-repeat;
     text-indent: 0;
     height: 31px;
-    font-size: 18px;
+    font-size: 16px;
-    font-weight: bold;
+/*    font-weight: bold;*/
     padding-left: 50px;
     line-height: 30px;
 }
 
 #branding h1, #branding h1 a:link, #branding h1 a:visited {
-    color: #707070;
+    color: #555;
 }
 
 

orchestra/static/orchestra/images/orchestra-logo.svg

@@ -16,8 +16,8 @@
    inkscape:version="0.48.3.1 r9886"
    sodipodi:docname="orchestra-logo.svg"
    inkscape:export-filename="/home/glic3rinu/orchestra/django-orchestra/orchestra/static/orchestra/images/orchestra-logo.png"
-   inkscape:export-xdpi="90"
+   inkscape:export-xdpi="81.290321"
-   inkscape:export-ydpi="90">
+   inkscape:export-ydpi="81.290321">
   <defs
      id="defs4">
     <linearGradient

orchestra/templatetags/utils.py

@@ -9,7 +9,7 @@ register = template.Library()
 
 
 @register.simple_tag(name="version")
-def controller_version():
+def orchestra_version():
     return get_version()
 
 

orchestra/utils/options.py

@@ -1,3 +1,4 @@
+import sys
 import urlparse
 
 from django.core.mail import EmailMultiAlternatives
@@ -37,3 +38,6 @@ def send_email_template(template, context, to, email_from=None, html=None):
         msg.attach_alternative(html_message, "text/html")
     msg.send()
 
+
+def running_syncdb():
+    return 'migrate' in sys.argv or 'syncdb' in sys.argv