Added support for service handlers

orchestra/apps/accounts/admin.py

@@ -61,7 +61,7 @@ class AccountAdmin(ExtendedModelAdmin):
                 messages.warning(request, 'This account is disabled.')
         context = {
             'services': sorted(
-                [ model._meta for model in services.get().keys() ],
+                [ model._meta for model in services.get() ],
                 key=lambda i: i.verbose_name_plural.lower()
             )
         }

orchestra/apps/orchestration/backends.py

@@ -8,7 +8,7 @@ from orchestra.utils import plugins
 from . import methods
 
 
-class ServiceBackend(object):
+class ServiceBackend(plugins.Plugin):
     """
     Service management backend base class
     
@@ -16,7 +16,6 @@ class ServiceBackend(object):
     be conviniently supported. Each backend generates the configuration for all
     the changes of all modified objects, reloading the daemon just once.
     """
-    verbose_name = None
     model = None
     related_models = () # ((model, accessor__attribute),)
     script_method = methods.BashSSH
@@ -65,28 +64,11 @@ class ServiceBackend(object):
     
     @classmethod
     def get_backends(cls):
-        return cls.plugins
+        return cls.get_plugins()
     
     @classmethod
     def get_backend(cls, name):
-        for backend in ServiceBackend.get_backends():
+        return cls.get_plugin(name)
-            if backend.get_name() == name:
-                return backend
-        raise KeyError('This backend is not registered')
-    
-    @classmethod
-    def get_choices(cls):
-        backends = cls.get_backends()
-        choices = []
-        for b in backends:
-            # don't evaluate b.verbose_name ugettext_lazy
-            verbose = getattr(b.verbose_name, '_proxy____args', [b.verbose_name])
-            if verbose[0]:
-                verbose = b.verbose_name
-            else:
-                verbose = b.get_name()
-            choices.append((b.get_name(), verbose))
-        return sorted(choices, key=lambda e: e[0])
     
     def get_banner(self):
         time = timezone.now().strftime("%h %d, %Y %I:%M:%S")

orchestra/apps/orchestration/models.py

@@ -125,7 +125,7 @@ class Route(models.Model):
     Defines the routing that determine in which server a backend is executed
     """
     backend = models.CharField(_("backend"), max_length=256,
-            choices=ServiceBackend.get_choices())
+            choices=ServiceBackend.get_plugin_choices())
     host = models.ForeignKey(Server, verbose_name=_("host"))
     match = models.CharField(_("match"), max_length=256, blank=True, default='True',
             help_text=_("Python expression used for selecting the targe host, "

orchestra/apps/orders/admin.py

@@ -12,7 +12,7 @@ class ServiceAdmin(admin.ModelAdmin):
     fieldsets = (
         (None, {
             'classes': ('wide',),
-            'fields': ('description', 'model', 'match', 'is_active')
+            'fields': ('description', 'content_type', 'match', 'handler', 'is_active')
         }),
         (_("Billing options"), {
             'classes': ('wide',),
@@ -22,16 +22,17 @@ class ServiceAdmin(admin.ModelAdmin):
         (_("Pricing options"), {
             'classes': ('wide',),
             'fields': ('metric', 'pricing_period', 'rate_algorithm',
-                       'orders_effect', ('on_cancel', 'on_disable', 'on_register'),
+                       'orders_effect', 'on_cancel', 'payment_style',
-                       'payment_style', 'trial_period', 'refound_period', 'tax',)
+                       'trial_period', 'refound_period', 'tax')
         }),
     )
     
     def formfield_for_dbfield(self, db_field, **kwargs):
         """ Improve performance of account field and filter by account """
-        if db_field.name == 'model':
+        if db_field.name == 'content_type':
-            models = [model._meta.model_name for model in services.get().keys()]
+            models = [model._meta.model_name for model in services.get()]
-            kwargs['queryset'] = db_field.rel.to.objects.filter(model__in=models)
+            queryset = db_field.rel.to.objects
+            kwargs['queryset'] = queryset.filter(model__in=models)
         if db_field.name in ['match', 'metric']:
             kwargs['widget'] = forms.TextInput(attrs={'size':'160'})
         return super(ServiceAdmin, self).formfield_for_dbfield(db_field, **kwargs)
@@ -43,7 +44,7 @@ class OrderAdmin(AccountAdminMixin, admin.ModelAdmin):
 
 
 class MetricStorageAdmin(admin.ModelAdmin):
-    list_display = ('order', 'value', 'date')
+    list_display = ('order', 'value', 'created_on', 'updated_on')
     list_filter = ('order__service',)
 
 

orchestra/apps/orders/handlers.py

@@ -0,0 +1,39 @@
+from django.contrib.contenttypes.models import ContentType
+from django.utils.translation import ugettext_lazy as _
+
+from orchestra.utils import plugins
+
+
+class ServiceHandler(plugins.Plugin):
+    model = None
+    
+    __metaclass__ = plugins.PluginMount
+    
+    def __init__(self, service):
+        self.service = service
+    
+    @classmethod
+    def get_plugin_choices(cls):
+        choices = super(ServiceHandler, cls).get_plugin_choices()
+        return [('', _("Default"))] + choices
+    
+    def __getattr__(self, attr):
+        return getattr(self.service, attr)
+    
+    def matches(self, instance):
+        safe_locals = {
+            instance._meta.model_name: instance
+        }
+        return eval(self.match, safe_locals)
+    
+    def get_metric(self, instance):
+        safe_locals = {
+            instance._meta.model_name: instance
+        }
+        return eval(self.metric, safe_locals)
+    
+    def get_content_type(self):
+        if not self.model:
+            return self.content_type
+        app_label, model = self.model.split('.')
+        return ContentType.objects.get_by_natural_key(app_label, model.lower())

orchestra/apps/orders/models.py

@@ -5,13 +5,22 @@ from django.dispatch import receiver
 from django.contrib.admin.models import LogEntry
 from django.contrib.contenttypes import generic
 from django.contrib.contenttypes.models import ContentType
+from django.core.validators import ValidationError
 from django.utils import timezone
+from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
+from orchestra.core import caches
+from orchestra.utils.apps import autodiscover
+
 from . import settings
+from .handlers import ServiceHandler
 from .helpers import search_for_related
 
 
+autodiscover('handlers')
+
+
 class Service(models.Model):
     NEVER = 'NEVER'
     MONTHLY = 'MONTHLY'
@@ -34,10 +43,12 @@ class Service(models.Model):
     MATCH_PRICE = 'MATCH_PRICE'
     
     description = models.CharField(_("description"), max_length=256, unique=True)
-    model = models.ForeignKey(ContentType, verbose_name=_("model"))
+    content_type = models.ForeignKey(ContentType, verbose_name=_("content type"))
-    match = models.CharField(_("match"), max_length=256)
+    match = models.CharField(_("match"), max_length=256, blank=True)
+    handler = models.CharField(_("handler"), max_length=256, blank=True,
+            help_text=_("Handler used to process this Service."),
+            choices=ServiceHandler.get_plugin_choices())
     is_active = models.BooleanField(_("is active"), default=True)
-    # TODO class based Service definition (like ServiceBackend)
     # Billing
     billing_period = models.CharField(_("billing period"), max_length=16,
             help_text=_("Renewal period for recurring invoicing"),
@@ -105,21 +116,22 @@ class Service(models.Model):
                 (REFOUND, _("Refound")),
             ),
             default=DISCOUNT)
-    on_disable = models.CharField(_("on disable"), max_length=16,
+    # TODO remove, orders are not disabled (they are cancelled user.is_active)
-            help_text=_("Defines the behaviour of this service when disabled"),
+#    on_disable = models.CharField(_("on disable"), max_length=16,
-            choices=(
+#            help_text=_("Defines the behaviour of this service when disabled"),
-                (NOTHING, _("Nothing")),
+#            choices=(
-                (DISCOUNT, _("Discount")),
+#                (NOTHING, _("Nothing")),
-                (REFOUND, _("Refound")),
+#                (DISCOUNT, _("Discount")),
-            ),
+#                (REFOUND, _("Refound")),
-            default=DISCOUNT)
+#            ),
-    on_register = models.CharField(_("on register"), max_length=16,
+#            default=DISCOUNT)
-            help_text=_("Defines the behaviour of this service on registration"),
+#    on_register = models.CharField(_("on register"), max_length=16,
-            choices=(
+#            help_text=_("Defines the behaviour of this service on registration"),
-                (NOTHING, _("Nothing")),
+#            choices=(
-                (DISCOUNT, _("Discount (fixed BP)")),
+#                (NOTHING, _("Nothing")),
-            ),
+#                (DISCOUNT, _("Discount (fixed BP)")),
-            default=DISCOUNT)
+#            ),
+#            default=DISCOUNT)
     payment_style = models.CharField(_("payment style"), max_length=16,
             help_text=_("Designates whether this service should be paid after "
                         "consumtion (postpay/on demand) or prepaid"),
@@ -151,27 +163,38 @@ class Service(models.Model):
         return self.description
     
     @classmethod
-    def get_services(cls, instance, **kwargs):
+    def get_services(cls, instance):
-        # TODO get per-request cache from thread local
+        cache = caches.get_request_cache()
-        cache = kwargs.get('cache', {})
         ct = ContentType.objects.get_for_model(instance)
+        services = cache.get(ct)
+        if services is None:
+            services = cls.objects.filter(content_type=ct, is_active=True)
+            cache.set(ct, services)
+        return services
+    
+    @cached_property
+    def proxy(self):
+        if self.handler:
+            return ServiceHandler.get_plugin(self.handler)(self)
+        return ServiceHandler(self)
+    
+    def clean(self):
+        content_type = self.proxy.get_content_type()
+        if self.content_type != content_type:
+            msg =_("Content type must be equal to '%s'." % str(content_type))
+            raise ValidationError(msg)
+        if not self.match:
+            msg =_("Match should be provided")
+            raise ValidationError(msg)
         try:
-            return cache[ct]
+            obj = content_type.model_class().objects.all()[0]
-        except KeyError:
+        except IndexError:
-            cache[ct] = cls.objects.filter(model=ct, is_active=True)
+            pass
-            return cache[ct]
+        else:
-    
+            try:
-    def matches(self, instance):
+                self.proxy.matches(obj)
-        safe_locals = {
+            except Exception as e:
-            instance._meta.model_name: instance
+                raise ValidationError(_(str(e)))
-        }
-        return eval(self.match, safe_locals)
-    
-    def get_metric(self, instance):
-        safe_locals = {
-            instance._meta.model_name: instance
-        }
-        return eval(self.metric, safe_locals)
 
 
 class OrderQuerySet(models.QuerySet):
@@ -242,10 +265,11 @@ class Order(models.Model):
 class MetricStorage(models.Model):
     order = models.ForeignKey(Order, verbose_name=_("order"))
     value = models.BigIntegerField(_("value"))
-    date = models.DateTimeField(_("date"), auto_now_add=True)
+    created_on = models.DateTimeField(_("created on"), auto_now_add=True)
+    updated_on = models.DateTimeField(_("updated on"), auto_now=True)
     
     class Meta:
-        get_latest_by = 'date'
+        get_latest_by = 'created_on'
     
     def __unicode__(self):
         return unicode(self.order)
@@ -259,24 +283,29 @@ class MetricStorage(models.Model):
         else:
             if metric.value != value:
                 cls.objects.create(order=order, value=value)
+            else:
+                metric.save()
 
 
 @receiver(pre_delete, dispatch_uid="orders.cancel_orders")
 def cancel_orders(sender, **kwargs):
-    if not sender in [MetricStorage, LogEntry, Order, Service]:
+    if (not sender in [MetricStorage, LogEntry, Order, Service] and
-        instance = kwargs['instance']
+        not Service in sender.__mro__):
-        for order in Order.objects.by_object(instance).active():
+            instance = kwargs['instance']
-            order.cancel()
+            for order in Order.objects.by_object(instance).active():
+                order.cancel()
 
 
 @receiver(post_save, dispatch_uid="orders.update_orders")
 @receiver(post_delete, dispatch_uid="orders.update_orders")
 def update_orders(sender, **kwargs):
-    if not sender in [MetricStorage, LogEntry, Order, Service]:
+    if (not sender in [MetricStorage, LogEntry, Order, Service] and
-        instance = kwargs['instance']
+        not Service in sender.__mro__):
-        if instance.pk:
+            instance = kwargs['instance']
-            # post_save
+            print kwargs
-            Order.update_orders(instance)
+            if instance.pk:
-        related = search_for_related(instance)
+                # post_save
-        if related:
+                Order.update_orders(instance)
-            Order.update_orders(related)
+            related = search_for_related(instance)
+            if related:
+                Order.update_orders(related)

orchestra/apps/resources/admin.py

@@ -59,7 +59,7 @@ class ResourceAdmin(ExtendedModelAdmin):
     def formfield_for_dbfield(self, db_field, **kwargs):
         """ filter service content_types """
         if db_field.name == 'content_type':
-            models = [ model._meta.model_name for model in services.get().keys() ]
+            models = [ model._meta.model_name for model in services.get() ]
             kwargs['queryset'] = db_field.rel.to.objects.filter(model__in=models)
         return super(ResourceAdmin, self).formfield_for_dbfield(db_field, **kwargs)
 

orchestra/apps/resources/backends.py

@@ -31,7 +31,7 @@ class ServiceMonitor(ServiceBackend):
     def content_type(self):
         app_label, model = self.model.split('.')
         model = model.lower()
-        return ContentType.objects.get(app_label=app_label, model=model)
+        return ContentType.objects.get_by_natural_key(app_label, model)
     
     def get_last_data(self, object_id):
         from .models import MonitorData
@@ -56,7 +56,7 @@ class ServiceMonitor(ServiceBackend):
         from .models import MonitorData
         name = self.get_name()
         app_label, model_name = self.model.split('.')
-        ct = ContentType.objects.get(app_label=app_label, model=model_name.lower())
+        ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower())
         for line in log.stdout.splitlines():
             line = line.strip()
             object_id, value = self.process(line)

orchestra/apps/resources/models.py

@@ -58,7 +58,7 @@ class Resource(models.Model):
             help_text=_("Crontab for periodic execution. "
                         "Leave it empty to disable periodic monitoring"))
     monitors = MultiSelectField(_("monitors"), max_length=256, blank=True,
-            choices=ServiceMonitor.get_choices(),
+            choices=ServiceMonitor.get_plugin_choices(),
             help_text=_("Monitor backends used for monitoring this resource."))
     is_active = models.BooleanField(_("is active"), default=True)
     
@@ -144,7 +144,7 @@ class ResourceData(models.Model):
 class MonitorData(models.Model):
     """ Stores monitored data """
     monitor = models.CharField(_("monitor"), max_length=256,
-            choices=ServiceMonitor.get_choices())
+            choices=ServiceMonitor.get_plugin_choices())
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
     date = models.DateTimeField(_("date"), auto_now_add=True)

orchestra/conf/base_settings.py

@@ -43,6 +43,7 @@ MIDDLEWARE_CLASSES = (
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.transaction.TransactionMiddleware',
+    'orchestra.core.cache.RequestCacheMiddleware',
     'orchestra.apps.orchestration.middlewares.OperationsMiddleware',
     # Uncomment the next line for simple clickjacking protection:
     # 'django.middleware.clickjacking.XFrameOptionsMiddleware',

orchestra/core/caches.py

@@ -0,0 +1,39 @@
+from threading import currentThread
+
+from django.core.cache.backends.locmem import LocMemCache
+
+
+_request_cache = {}
+
+
+class RequestCache(LocMemCache):
+    """ LocMemCache is a threadsafe local memory cache """
+    def __init__(self):
+        name = 'locmemcache@%i' % hash(currentThread())
+        super(RequestCache, self).__init__(name, {})
+
+
+def get_request_cache():
+    """
+    Returns per-request cache when running RequestCacheMiddleware otherwise a
+    new LocMemCache instance (when running periodic tasks or shell)
+    """
+    try:
+        return _request_cache[currentThread()]
+    except KeyError:
+        cache = RequestCache()
+        _request_cache[currentThread()] = cache
+        return cache
+
+
+class RequestCacheMiddleware(object):
+    def process_request(self, request):
+        cache = _request_cache.get(currentThread(), RequestCache())
+        _request_cache[currentThread()] = cache
+        cache.clear()
+    
+    def process_response(self, request, response):
+        # TODO not sure if this actually saves memory, remove otherwise
+        if currentThread() in _request_cache:
+            _request_cache[currentThread()].clear()
+        return response

orchestra/utils/functional.py

@@ -2,8 +2,15 @@ def cached(func):
     """ caches func return value """
     def cached_func(self, *args, **kwargs):
         attr = '_cached_' + func.__name__
-        if not hasattr(self, attr):
+        key = (args, tuple(kwargs.items()))
-            setattr(self, attr, func(self, *args, **kwargs))
+        try:
-        return getattr(self, attr)
+            return getattr(self, attr)[key]
+        except KeyError:
+            value = func(self, *args, **kwargs)
+            getattr(self, attr)[key] = value
+        except AttributeError:
+            value = func(self, *args, **kwargs)
+            setattr(self, attr, {key: value})
+        return value
     return cached_func
 

orchestra/utils/plugins.py

@@ -1,3 +1,40 @@
+from .functional import cached
+
+
+class Plugin(object):
+    verbose_name = None
+    
+    @classmethod
+    def get_plugin_name(cls):
+        return cls.__name__
+    
+    @classmethod
+    def get_plugins(cls):
+        return cls.plugins
+    
+    @classmethod
+    @cached
+    def get_plugin(cls, name):
+        for plugin in cls.get_plugins():
+            if plugin.get_plugin_name() == name:
+                return plugin
+        raise KeyError('This plugin is not registered')
+    
+    @classmethod
+    def get_plugin_choices(cls):
+        plugins = cls.get_plugins()
+        choices = []
+        for p in plugins:
+            # don't evaluate p.verbose_name ugettext_lazy
+            verbose = getattr(p.verbose_name, '_proxy____args', [p.verbose_name])
+            if verbose[0]:
+                verbose = p.verbose_name
+            else:
+                verbose = p.get_plugin_name()
+            choices.append((p.get_plugin_name(), verbose))
+        return sorted(choices, key=lambda e: e[0])
+
+
 class PluginMount(type):
     def __init__(cls, name, bases, attrs):
         if not hasattr(cls, 'plugins'):