Added advance orchestration functionality

orchestra/contrib/orchestration/actions.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 from django.contrib import messages
 from django.contrib.admin import helpers
 from django.shortcuts import render
@@ -6,9 +8,10 @@ from django.utils.translation import ungettext, ugettext_lazy as _
 
 from orchestra.admin.utils import get_object_from_url, change_url
 from orchestra.contrib.orchestration.helpers import message_user
+from orchestra.utils.python import OrderedSet
 
-from . import Operation
-from .models import BackendOperation
+from . import manager, Operation
+from .models import BackendOperation, Route, Server
 
 
 def retry_backend(modeladmin, request, queryset):
@@ -26,7 +29,6 @@ def retry_backend(modeladmin, request, queryset):
         else:
             logs = Operation.execute(operations)
             message_user(request, logs)
-        Operation.execute(operations)
         return
     opts = modeladmin.model._meta
     display_objects = []
@@ -61,3 +63,66 @@ def retry_backend(modeladmin, request, queryset):
     return render(request, 'admin/orchestration/backends/retry.html', context)
 retry_backend.short_description = _("Retry")
 retry_backend.url_name = 'retry'
+
+
+def orchestrate(modeladmin, request, queryset):
+    operations = set()
+    action = Operation.SAVE
+    operations = OrderedSet()
+    if queryset.model is Route:
+        for route in queryset:
+            routes = [route]
+            backend = route.backend_class
+            if action not in backend.actions:
+                continue
+            for instance in backend.model_class().objects.all():
+                if route.matches(instance):
+                    operations.add(Operation(backend, instance, action, routes=routes))
+    elif queryset.model is Server:
+        models = set()
+        for server in queryset:
+            routes = server.routes.all()
+            for route in routes.filter(is_active=True):
+                model = route.backend_class.model_class()
+                models.add(model)
+        querysets = [model.objects.order_by('id') for model in models]
+        
+        route_cache = {}
+        for model in models:
+            for instance in model.objects.all():
+                manager.collect(instance, action, operations=operations, route_cache=route_cache)
+            routes = []
+        result = []
+        for operation in operations:
+            routes = [route for route in operation.routes if route.host in queryset]
+            operation.routes = routes
+            if routes:
+                result.append(operation)
+        operations = result
+    if not operations:
+        messages.warning(request, _("No operations."))
+    
+    if request.POST.get('post') == 'generic_confirmation':
+        logs = Operation.execute(operations)
+        message_user(request, logs)
+        return
+    
+    opts = modeladmin.model._meta
+    display_objects = {}
+    for operation in operations:
+        try:
+            display_objects[operation.backend].append(operation)
+        except KeyError:
+            display_objects[operation.backend] = [operation]
+    context = {
+        'title': _("Are you sure to execute the following operations?"),
+        'action_name': _('Orchestrate'),
+        'action_value': 'orchestrate',
+        'display_objects': display_objects,
+        'queryset': queryset,
+        'opts': opts,
+        'app_label': opts.app_label,
+        'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
+        'obj': get_object_from_url(modeladmin, request),
+    }
+    return render(request, 'admin/orchestration/orchestrate.html', context)

orchestra/contrib/orchestration/admin.py

@@ -8,7 +8,7 @@ from orchestra.admin.utils import admin_link, admin_date, admin_colored, display
 from orchestra.plugins.admin import display_plugin_field
 
 from . import settings, helpers
-from .actions import retry_backend
+from .actions import retry_backend, orchestrate
 from .backends import ServiceBackend
 from .forms import RouteForm
 from .models import Server, Route, BackendLog, BackendOperation
@@ -39,6 +39,7 @@ class RouteAdmin(ExtendedModelAdmin):
     ordering = ('backend',)
     add_fields = ('backend', 'host', 'match', 'async', 'is_active')
     change_form = RouteForm
+    actions = (orchestrate,)
     
     BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends())
     DEFAULT_MATCH = {
@@ -173,6 +174,7 @@ class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin):
 class ServerAdmin(admin.ModelAdmin):
     list_display = ('name', 'address', 'os', 'display_ping', 'display_uptime')
     list_filter = ('os',)
+    actions = (orchestrate,)
     
     def display_ping(self, instance):
         return self._remote_state[instance.pk][0]

orchestra/contrib/orchestration/management/commands/orchestrate.py

@@ -1,11 +1,11 @@
 import time
 from django.core.management.base import BaseCommand, CommandError
-from django.apps import apps
+from django.db.models import Q
 
 from orchestra.contrib.orchestration import manager, Operation
 from orchestra.contrib.orchestration.models import Server
 from orchestra.contrib.orchestration.backends import ServiceBackend
-from orchestra.utils.python import import_class, OrderedSet, AttrDict
+from orchestra.utils.python import OrderedSet
 from orchestra.utils.sys import confirm
 
 
@@ -13,70 +13,90 @@ class Command(BaseCommand):
     help = 'Runs orchestration backends.'
     
     def add_arguments(self, parser):
-        parser.add_argument('model',
+        parser.add_argument('model', nargs='?',
             help='Label of a model to execute the orchestration.')
         parser.add_argument('query', nargs='*',
             help='Query arguments for filter().')
         parser.add_argument('--noinput', action='store_false', dest='interactive', default=True,
             help='Tells Django to NOT prompt the user for input of any kind.')
-        parser.add_argument('--action', action='store', dest='action',
+        parser.add_argument('-a', '--action', action='store', dest='action',
             default='save', help='Executes action. Defaults to "save".')
-        parser.add_argument('--servers', action='store', dest='servers',
+        parser.add_argument('-s', '--servers', action='store', dest='servers',
             default='', help='Overrides route server resolution with the provided server.')
-        parser.add_argument('--backends', action='store', dest='backends',
+        parser.add_argument('-b', '--backends', action='store', dest='backends',
             default='', help='Overrides backend.')
-        parser.add_argument('--listbackends', action='store_true', dest='list_backends', default=False,
+        parser.add_argument('-l', '--listbackends', action='store_true', dest='list_backends', default=False,
             help='List available baclends.')
         parser.add_argument('--dry-run', action='store_true', dest='dry', default=False,
             help='Only prints scrtipt.')
     
+    
+    def collect_operations(self, **options):
+        model = options.get('model')
+        backends = options.get('backends') or set()
+        if backends:
+            backends = set(backends.split(','))
+        servers = options.get('servers') or set()
+        if servers:
+            servers = set([Server.objects.get(Q(address=server)|Q(name=server)) for server in servers.split(',')])
+        action = options.get('action')
+        if not model:
+            models = set()
+            if servers:
+                for server in servers:
+                    if backends:
+                        routes = server.routes.filter(backend__in=backends)
+                    else:
+                        routes = server.routes.all()
+            elif backends:
+                routes = Route.objects.filter(backend__in=backends)
+            else:
+                raise CommandError("Model or --servers or --backends?")
+            for route in routes.filter(is_active=True):
+                model = route.backend_class.model_class()
+                models.add(model)
+            querysets = [model.objects.order_by('id') for model in models]
+        else:
+            kwargs = {}
+            for comp in options.get('query', []):
+                comps = iter(comp.split('='))
+                for arg in comps:
+                    kwargs[arg] = next(comps).strip().rstrip(',')
+            model = apps.get_model(*model.split('.'))
+            queryset = model.objects.filter(**kwargs).order_by('id')
+            querysets = [queryset]
+        
+        operations = OrderedSet()
+        route_cache = {}
+        for queryset in querysets:
+            for instance in queryset:
+                manager.collect(instance, action, operations=operations, route_cache=route_cache)
+        if backends:
+            result = []
+            for operation in operations:
+                if operation.backend in backends:
+                    result.append(operation)
+            operations = result
+        if servers:
+            routes = []
+            result = []
+            for operation in operations:
+                routes = [route for route in operation.routes if route.host in servers]
+                operation.routes = routes
+                if routes:
+                    result.append(operation)
+            operations = result
+        return operations
+    
     def handle(self, *args, **options):
         list_backends = options.get('list_backends')
         if list_backends:
             for backend in ServiceBackend.get_backends():
                 self.stdout.write(str(backend).split("'")[1])
             return
-        model = apps.get_model(*options['model'].split('.'))
-        action = options.get('action')
         interactive = options.get('interactive')
-        servers = options.get('servers')
-        backends = options.get('backends')
-        if (servers and not backends) or (not servers and backends):
-            raise CommandError("--backends and --servers go in tandem.")
         dry = options.get('dry')
-        kwargs = {}
-        for comp in options.get('query', []):
-            comps = iter(comp.split('='))
-            for arg in comps:
-                kwargs[arg] = next(comps).strip().rstrip(',')
-        operations = OrderedSet()
-        route_cache = {}
-        queryset = model.objects.filter(**kwargs).order_by('id')
-        if servers:
-            servers = servers.split(',')
-            backends = backends.split(',')
-            routes = []
-            # Get and create missing Servers
-            for server in servers:
-                try:
-                    server = Server.objects.get(address=server)
-                except Server.DoesNotExist:
-                    server = Server(name=server, address=server)
-                    server.full_clean()
-                    server.save()
-                routes.append(AttrDict(
-                    host=server,
-                    async=False,
-                    action_is_async=lambda self: False,
-                ))
-            # Generate operations for the given backend
-            for instance in queryset:
-                for backend in backends:
-                    backend = import_class(backend)
-                    operations.add(Operation(backend, instance, action, routes=routes))
-        else:
-            for instance in queryset:
-                manager.collect(instance, action, operations=operations, route_cache=route_cache)
+        operations = self.collect_operations(**options)
         scripts, serialize = manager.generate(operations)
         servers = set()
         # Print scripts

orchestra/contrib/orchestration/models.py

@@ -199,7 +199,7 @@ class Route(models.Model):
     """
     backend = models.CharField(_("backend"), max_length=256,
         choices=ServiceBackend.get_choices())
-    host = models.ForeignKey(Server, verbose_name=_("host"))
+    host = models.ForeignKey(Server, verbose_name=_("host"), related_name='routes')
     match = models.CharField(_("match"), max_length=256, blank=True, default='True',
         help_text=_("Python expression used for selecting the targe host, "
                     "<em>instance</em> referes to the current object."))

orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html

@@ -0,0 +1,17 @@
+{% extends "admin/orchestra/generic_confirmation.html" %}
+{% load utils %}
+
+{% block display_objects %}
+    <ul>
+    {% for backend, operations in display_objects.items %}
+        <li>{{ backend }}
+        <ul>
+            {% for operation in operations %}
+              <li><a href="{{ operation.instance|admin_url }}">{{ operation.instance }}</a>:
+            into {% for route in operation.routes %}<a href="{{ route.host|admin_url }}">{{ route.host }}</a>{% if not forloop.last %},{% endif %} {% endfor %}</li>
+            {% endfor %}
+        </ul>
+        </li>
+    {% endfor %}
+    </ul>
+{% endblock %}

orchestra/templates/admin/orchestra/generic_confirmation.html

@@ -29,7 +29,9 @@
   <div>
     <div style="margin:20px;">
     <p>{{ content_message | safe }}</p>
+    {% block display_objects %}
         <ul>{{ display_objects | unordered_list }}</ul>
+    {% endblock %}
     <form action="" method="post">{% csrf_token %}
     {% block form %}
     {% if form %}

orchestra/templatetags/utils.py

@@ -11,7 +11,7 @@ from django.template.defaultfilters import date
 from django.utils.safestring import mark_safe
 
 from orchestra import get_version
-from orchestra.admin.utils import change_url
+from orchestra.admin.utils import change_url, admin_link as utils_admin_link
 from orchestra.utils.apps import isinstalled