Added advance orchestration functionality

This commit is contained in:
Marc Aymerich 2016-07-15 12:44:38 +00:00
parent 6ca38f092b
commit df99f8d745
7 changed files with 159 additions and 53 deletions

View File

@ -1,3 +1,5 @@
from collections import defaultdict
from django.contrib import messages
from django.contrib.admin import helpers
from django.shortcuts import render
@ -6,9 +8,10 @@ from django.utils.translation import ungettext, ugettext_lazy as _
from orchestra.admin.utils import get_object_from_url, change_url
from orchestra.contrib.orchestration.helpers import message_user
from orchestra.utils.python import OrderedSet
from . import Operation
from .models import BackendOperation
from . import manager, Operation
from .models import BackendOperation, Route, Server
def retry_backend(modeladmin, request, queryset):
@ -26,7 +29,6 @@ def retry_backend(modeladmin, request, queryset):
else:
logs = Operation.execute(operations)
message_user(request, logs)
Operation.execute(operations)
return
opts = modeladmin.model._meta
display_objects = []
@ -61,3 +63,66 @@ def retry_backend(modeladmin, request, queryset):
return render(request, 'admin/orchestration/backends/retry.html', context)
retry_backend.short_description = _("Retry")
retry_backend.url_name = 'retry'
def orchestrate(modeladmin, request, queryset):
operations = set()
action = Operation.SAVE
operations = OrderedSet()
if queryset.model is Route:
for route in queryset:
routes = [route]
backend = route.backend_class
if action not in backend.actions:
continue
for instance in backend.model_class().objects.all():
if route.matches(instance):
operations.add(Operation(backend, instance, action, routes=routes))
elif queryset.model is Server:
models = set()
for server in queryset:
routes = server.routes.all()
for route in routes.filter(is_active=True):
model = route.backend_class.model_class()
models.add(model)
querysets = [model.objects.order_by('id') for model in models]
route_cache = {}
for model in models:
for instance in model.objects.all():
manager.collect(instance, action, operations=operations, route_cache=route_cache)
routes = []
result = []
for operation in operations:
routes = [route for route in operation.routes if route.host in queryset]
operation.routes = routes
if routes:
result.append(operation)
operations = result
if not operations:
messages.warning(request, _("No operations."))
if request.POST.get('post') == 'generic_confirmation':
logs = Operation.execute(operations)
message_user(request, logs)
return
opts = modeladmin.model._meta
display_objects = {}
for operation in operations:
try:
display_objects[operation.backend].append(operation)
except KeyError:
display_objects[operation.backend] = [operation]
context = {
'title': _("Are you sure to execute the following operations?"),
'action_name': _('Orchestrate'),
'action_value': 'orchestrate',
'display_objects': display_objects,
'queryset': queryset,
'opts': opts,
'app_label': opts.app_label,
'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
'obj': get_object_from_url(modeladmin, request),
}
return render(request, 'admin/orchestration/orchestrate.html', context)

View File

@ -8,7 +8,7 @@ from orchestra.admin.utils import admin_link, admin_date, admin_colored, display
from orchestra.plugins.admin import display_plugin_field
from . import settings, helpers
from .actions import retry_backend
from .actions import retry_backend, orchestrate
from .backends import ServiceBackend
from .forms import RouteForm
from .models import Server, Route, BackendLog, BackendOperation
@ -39,6 +39,7 @@ class RouteAdmin(ExtendedModelAdmin):
ordering = ('backend',)
add_fields = ('backend', 'host', 'match', 'async', 'is_active')
change_form = RouteForm
actions = (orchestrate,)
BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends())
DEFAULT_MATCH = {
@ -173,6 +174,7 @@ class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin):
class ServerAdmin(admin.ModelAdmin):
list_display = ('name', 'address', 'os', 'display_ping', 'display_uptime')
list_filter = ('os',)
actions = (orchestrate,)
def display_ping(self, instance):
return self._remote_state[instance.pk][0]

View File

@ -1,11 +1,11 @@
import time
from django.core.management.base import BaseCommand, CommandError
from django.apps import apps
from django.db.models import Q
from orchestra.contrib.orchestration import manager, Operation
from orchestra.contrib.orchestration.models import Server
from orchestra.contrib.orchestration.backends import ServiceBackend
from orchestra.utils.python import import_class, OrderedSet, AttrDict
from orchestra.utils.python import OrderedSet
from orchestra.utils.sys import confirm
@ -13,70 +13,90 @@ class Command(BaseCommand):
help = 'Runs orchestration backends.'
def add_arguments(self, parser):
parser.add_argument('model',
parser.add_argument('model', nargs='?',
help='Label of a model to execute the orchestration.')
parser.add_argument('query', nargs='*',
help='Query arguments for filter().')
parser.add_argument('--noinput', action='store_false', dest='interactive', default=True,
help='Tells Django to NOT prompt the user for input of any kind.')
parser.add_argument('--action', action='store', dest='action',
parser.add_argument('-a', '--action', action='store', dest='action',
default='save', help='Executes action. Defaults to "save".')
parser.add_argument('--servers', action='store', dest='servers',
parser.add_argument('-s', '--servers', action='store', dest='servers',
default='', help='Overrides route server resolution with the provided server.')
parser.add_argument('--backends', action='store', dest='backends',
parser.add_argument('-b', '--backends', action='store', dest='backends',
default='', help='Overrides backend.')
parser.add_argument('--listbackends', action='store_true', dest='list_backends', default=False,
parser.add_argument('-l', '--listbackends', action='store_true', dest='list_backends', default=False,
help='List available baclends.')
parser.add_argument('--dry-run', action='store_true', dest='dry', default=False,
help='Only prints scrtipt.')
def collect_operations(self, **options):
model = options.get('model')
backends = options.get('backends') or set()
if backends:
backends = set(backends.split(','))
servers = options.get('servers') or set()
if servers:
servers = set([Server.objects.get(Q(address=server)|Q(name=server)) for server in servers.split(',')])
action = options.get('action')
if not model:
models = set()
if servers:
for server in servers:
if backends:
routes = server.routes.filter(backend__in=backends)
else:
routes = server.routes.all()
elif backends:
routes = Route.objects.filter(backend__in=backends)
else:
raise CommandError("Model or --servers or --backends?")
for route in routes.filter(is_active=True):
model = route.backend_class.model_class()
models.add(model)
querysets = [model.objects.order_by('id') for model in models]
else:
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
model = apps.get_model(*model.split('.'))
queryset = model.objects.filter(**kwargs).order_by('id')
querysets = [queryset]
operations = OrderedSet()
route_cache = {}
for queryset in querysets:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
if backends:
result = []
for operation in operations:
if operation.backend in backends:
result.append(operation)
operations = result
if servers:
routes = []
result = []
for operation in operations:
routes = [route for route in operation.routes if route.host in servers]
operation.routes = routes
if routes:
result.append(operation)
operations = result
return operations
def handle(self, *args, **options):
list_backends = options.get('list_backends')
if list_backends:
for backend in ServiceBackend.get_backends():
self.stdout.write(str(backend).split("'")[1])
return
model = apps.get_model(*options['model'].split('.'))
action = options.get('action')
interactive = options.get('interactive')
servers = options.get('servers')
backends = options.get('backends')
if (servers and not backends) or (not servers and backends):
raise CommandError("--backends and --servers go in tandem.")
dry = options.get('dry')
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
operations = OrderedSet()
route_cache = {}
queryset = model.objects.filter(**kwargs).order_by('id')
if servers:
servers = servers.split(',')
backends = backends.split(',')
routes = []
# Get and create missing Servers
for server in servers:
try:
server = Server.objects.get(address=server)
except Server.DoesNotExist:
server = Server(name=server, address=server)
server.full_clean()
server.save()
routes.append(AttrDict(
host=server,
async=False,
action_is_async=lambda self: False,
))
# Generate operations for the given backend
for instance in queryset:
for backend in backends:
backend = import_class(backend)
operations.add(Operation(backend, instance, action, routes=routes))
else:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
operations = self.collect_operations(**options)
scripts, serialize = manager.generate(operations)
servers = set()
# Print scripts

View File

@ -199,7 +199,7 @@ class Route(models.Model):
"""
backend = models.CharField(_("backend"), max_length=256,
choices=ServiceBackend.get_choices())
host = models.ForeignKey(Server, verbose_name=_("host"))
host = models.ForeignKey(Server, verbose_name=_("host"), related_name='routes')
match = models.CharField(_("match"), max_length=256, blank=True, default='True',
help_text=_("Python expression used for selecting the targe host, "
"<em>instance</em> referes to the current object."))

View File

@ -0,0 +1,17 @@
{% extends "admin/orchestra/generic_confirmation.html" %}
{% load utils %}
{% block display_objects %}
<ul>
{% for backend, operations in display_objects.items %}
<li>{{ backend }}
<ul>
{% for operation in operations %}
<li><a href="{{ operation.instance|admin_url }}">{{ operation.instance }}</a>:
into {% for route in operation.routes %}<a href="{{ route.host|admin_url }}">{{ route.host }}</a>{% if not forloop.last %},{% endif %} {% endfor %}</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
{% endblock %}

View File

@ -29,7 +29,9 @@
<div>
<div style="margin:20px;">
<p>{{ content_message | safe }}</p>
{% block display_objects %}
<ul>{{ display_objects | unordered_list }}</ul>
{% endblock %}
<form action="" method="post">{% csrf_token %}
{% block form %}
{% if form %}

View File

@ -11,7 +11,7 @@ from django.template.defaultfilters import date
from django.utils.safestring import mark_safe
from orchestra import get_version
from orchestra.admin.utils import change_url
from orchestra.admin.utils import change_url, admin_link as utils_admin_link
from orchestra.utils.apps import isinstalled