Added advance orchestration functionality

This commit is contained in:
Marc Aymerich 2016-07-15 12:44:38 +00:00
parent 6ca38f092b
commit df99f8d745
7 changed files with 159 additions and 53 deletions

View file

@ -1,3 +1,5 @@
from collections import defaultdict
from django.contrib import messages from django.contrib import messages
from django.contrib.admin import helpers from django.contrib.admin import helpers
from django.shortcuts import render from django.shortcuts import render
@ -6,9 +8,10 @@ from django.utils.translation import ungettext, ugettext_lazy as _
from orchestra.admin.utils import get_object_from_url, change_url from orchestra.admin.utils import get_object_from_url, change_url
from orchestra.contrib.orchestration.helpers import message_user from orchestra.contrib.orchestration.helpers import message_user
from orchestra.utils.python import OrderedSet
from . import Operation from . import manager, Operation
from .models import BackendOperation from .models import BackendOperation, Route, Server
def retry_backend(modeladmin, request, queryset): def retry_backend(modeladmin, request, queryset):
@ -26,7 +29,6 @@ def retry_backend(modeladmin, request, queryset):
else: else:
logs = Operation.execute(operations) logs = Operation.execute(operations)
message_user(request, logs) message_user(request, logs)
Operation.execute(operations)
return return
opts = modeladmin.model._meta opts = modeladmin.model._meta
display_objects = [] display_objects = []
@ -61,3 +63,66 @@ def retry_backend(modeladmin, request, queryset):
return render(request, 'admin/orchestration/backends/retry.html', context) return render(request, 'admin/orchestration/backends/retry.html', context)
retry_backend.short_description = _("Retry") retry_backend.short_description = _("Retry")
retry_backend.url_name = 'retry' retry_backend.url_name = 'retry'
def orchestrate(modeladmin, request, queryset):
operations = set()
action = Operation.SAVE
operations = OrderedSet()
if queryset.model is Route:
for route in queryset:
routes = [route]
backend = route.backend_class
if action not in backend.actions:
continue
for instance in backend.model_class().objects.all():
if route.matches(instance):
operations.add(Operation(backend, instance, action, routes=routes))
elif queryset.model is Server:
models = set()
for server in queryset:
routes = server.routes.all()
for route in routes.filter(is_active=True):
model = route.backend_class.model_class()
models.add(model)
querysets = [model.objects.order_by('id') for model in models]
route_cache = {}
for model in models:
for instance in model.objects.all():
manager.collect(instance, action, operations=operations, route_cache=route_cache)
routes = []
result = []
for operation in operations:
routes = [route for route in operation.routes if route.host in queryset]
operation.routes = routes
if routes:
result.append(operation)
operations = result
if not operations:
messages.warning(request, _("No operations."))
if request.POST.get('post') == 'generic_confirmation':
logs = Operation.execute(operations)
message_user(request, logs)
return
opts = modeladmin.model._meta
display_objects = {}
for operation in operations:
try:
display_objects[operation.backend].append(operation)
except KeyError:
display_objects[operation.backend] = [operation]
context = {
'title': _("Are you sure to execute the following operations?"),
'action_name': _('Orchestrate'),
'action_value': 'orchestrate',
'display_objects': display_objects,
'queryset': queryset,
'opts': opts,
'app_label': opts.app_label,
'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
'obj': get_object_from_url(modeladmin, request),
}
return render(request, 'admin/orchestration/orchestrate.html', context)

View file

@ -8,7 +8,7 @@ from orchestra.admin.utils import admin_link, admin_date, admin_colored, display
from orchestra.plugins.admin import display_plugin_field from orchestra.plugins.admin import display_plugin_field
from . import settings, helpers from . import settings, helpers
from .actions import retry_backend from .actions import retry_backend, orchestrate
from .backends import ServiceBackend from .backends import ServiceBackend
from .forms import RouteForm from .forms import RouteForm
from .models import Server, Route, BackendLog, BackendOperation from .models import Server, Route, BackendLog, BackendOperation
@ -39,6 +39,7 @@ class RouteAdmin(ExtendedModelAdmin):
ordering = ('backend',) ordering = ('backend',)
add_fields = ('backend', 'host', 'match', 'async', 'is_active') add_fields = ('backend', 'host', 'match', 'async', 'is_active')
change_form = RouteForm change_form = RouteForm
actions = (orchestrate,)
BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends()) BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends())
DEFAULT_MATCH = { DEFAULT_MATCH = {
@ -173,6 +174,7 @@ class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin):
class ServerAdmin(admin.ModelAdmin): class ServerAdmin(admin.ModelAdmin):
list_display = ('name', 'address', 'os', 'display_ping', 'display_uptime') list_display = ('name', 'address', 'os', 'display_ping', 'display_uptime')
list_filter = ('os',) list_filter = ('os',)
actions = (orchestrate,)
def display_ping(self, instance): def display_ping(self, instance):
return self._remote_state[instance.pk][0] return self._remote_state[instance.pk][0]

View file

@ -1,11 +1,11 @@
import time import time
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from django.apps import apps from django.db.models import Q
from orchestra.contrib.orchestration import manager, Operation from orchestra.contrib.orchestration import manager, Operation
from orchestra.contrib.orchestration.models import Server from orchestra.contrib.orchestration.models import Server
from orchestra.contrib.orchestration.backends import ServiceBackend from orchestra.contrib.orchestration.backends import ServiceBackend
from orchestra.utils.python import import_class, OrderedSet, AttrDict from orchestra.utils.python import OrderedSet
from orchestra.utils.sys import confirm from orchestra.utils.sys import confirm
@ -13,70 +13,90 @@ class Command(BaseCommand):
help = 'Runs orchestration backends.' help = 'Runs orchestration backends.'
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('model', parser.add_argument('model', nargs='?',
help='Label of a model to execute the orchestration.') help='Label of a model to execute the orchestration.')
parser.add_argument('query', nargs='*', parser.add_argument('query', nargs='*',
help='Query arguments for filter().') help='Query arguments for filter().')
parser.add_argument('--noinput', action='store_false', dest='interactive', default=True, parser.add_argument('--noinput', action='store_false', dest='interactive', default=True,
help='Tells Django to NOT prompt the user for input of any kind.') help='Tells Django to NOT prompt the user for input of any kind.')
parser.add_argument('--action', action='store', dest='action', parser.add_argument('-a', '--action', action='store', dest='action',
default='save', help='Executes action. Defaults to "save".') default='save', help='Executes action. Defaults to "save".')
parser.add_argument('--servers', action='store', dest='servers', parser.add_argument('-s', '--servers', action='store', dest='servers',
default='', help='Overrides route server resolution with the provided server.') default='', help='Overrides route server resolution with the provided server.')
parser.add_argument('--backends', action='store', dest='backends', parser.add_argument('-b', '--backends', action='store', dest='backends',
default='', help='Overrides backend.') default='', help='Overrides backend.')
parser.add_argument('--listbackends', action='store_true', dest='list_backends', default=False, parser.add_argument('-l', '--listbackends', action='store_true', dest='list_backends', default=False,
help='List available baclends.') help='List available baclends.')
parser.add_argument('--dry-run', action='store_true', dest='dry', default=False, parser.add_argument('--dry-run', action='store_true', dest='dry', default=False,
help='Only prints scrtipt.') help='Only prints scrtipt.')
def collect_operations(self, **options):
model = options.get('model')
backends = options.get('backends') or set()
if backends:
backends = set(backends.split(','))
servers = options.get('servers') or set()
if servers:
servers = set([Server.objects.get(Q(address=server)|Q(name=server)) for server in servers.split(',')])
action = options.get('action')
if not model:
models = set()
if servers:
for server in servers:
if backends:
routes = server.routes.filter(backend__in=backends)
else:
routes = server.routes.all()
elif backends:
routes = Route.objects.filter(backend__in=backends)
else:
raise CommandError("Model or --servers or --backends?")
for route in routes.filter(is_active=True):
model = route.backend_class.model_class()
models.add(model)
querysets = [model.objects.order_by('id') for model in models]
else:
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
model = apps.get_model(*model.split('.'))
queryset = model.objects.filter(**kwargs).order_by('id')
querysets = [queryset]
operations = OrderedSet()
route_cache = {}
for queryset in querysets:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
if backends:
result = []
for operation in operations:
if operation.backend in backends:
result.append(operation)
operations = result
if servers:
routes = []
result = []
for operation in operations:
routes = [route for route in operation.routes if route.host in servers]
operation.routes = routes
if routes:
result.append(operation)
operations = result
return operations
def handle(self, *args, **options): def handle(self, *args, **options):
list_backends = options.get('list_backends') list_backends = options.get('list_backends')
if list_backends: if list_backends:
for backend in ServiceBackend.get_backends(): for backend in ServiceBackend.get_backends():
self.stdout.write(str(backend).split("'")[1]) self.stdout.write(str(backend).split("'")[1])
return return
model = apps.get_model(*options['model'].split('.'))
action = options.get('action')
interactive = options.get('interactive') interactive = options.get('interactive')
servers = options.get('servers')
backends = options.get('backends')
if (servers and not backends) or (not servers and backends):
raise CommandError("--backends and --servers go in tandem.")
dry = options.get('dry') dry = options.get('dry')
kwargs = {} operations = self.collect_operations(**options)
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
operations = OrderedSet()
route_cache = {}
queryset = model.objects.filter(**kwargs).order_by('id')
if servers:
servers = servers.split(',')
backends = backends.split(',')
routes = []
# Get and create missing Servers
for server in servers:
try:
server = Server.objects.get(address=server)
except Server.DoesNotExist:
server = Server(name=server, address=server)
server.full_clean()
server.save()
routes.append(AttrDict(
host=server,
async=False,
action_is_async=lambda self: False,
))
# Generate operations for the given backend
for instance in queryset:
for backend in backends:
backend = import_class(backend)
operations.add(Operation(backend, instance, action, routes=routes))
else:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
scripts, serialize = manager.generate(operations) scripts, serialize = manager.generate(operations)
servers = set() servers = set()
# Print scripts # Print scripts

View file

@ -199,7 +199,7 @@ class Route(models.Model):
""" """
backend = models.CharField(_("backend"), max_length=256, backend = models.CharField(_("backend"), max_length=256,
choices=ServiceBackend.get_choices()) choices=ServiceBackend.get_choices())
host = models.ForeignKey(Server, verbose_name=_("host")) host = models.ForeignKey(Server, verbose_name=_("host"), related_name='routes')
match = models.CharField(_("match"), max_length=256, blank=True, default='True', match = models.CharField(_("match"), max_length=256, blank=True, default='True',
help_text=_("Python expression used for selecting the targe host, " help_text=_("Python expression used for selecting the targe host, "
"<em>instance</em> referes to the current object.")) "<em>instance</em> referes to the current object."))

View file

@ -0,0 +1,17 @@
{% extends "admin/orchestra/generic_confirmation.html" %}
{% load utils %}
{% block display_objects %}
<ul>
{% for backend, operations in display_objects.items %}
<li>{{ backend }}
<ul>
{% for operation in operations %}
<li><a href="{{ operation.instance|admin_url }}">{{ operation.instance }}</a>:
into {% for route in operation.routes %}<a href="{{ route.host|admin_url }}">{{ route.host }}</a>{% if not forloop.last %},{% endif %} {% endfor %}</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
{% endblock %}

View file

@ -29,7 +29,9 @@
<div> <div>
<div style="margin:20px;"> <div style="margin:20px;">
<p>{{ content_message | safe }}</p> <p>{{ content_message | safe }}</p>
<ul>{{ display_objects | unordered_list }}</ul> {% block display_objects %}
<ul>{{ display_objects | unordered_list }}</ul>
{% endblock %}
<form action="" method="post">{% csrf_token %} <form action="" method="post">{% csrf_token %}
{% block form %} {% block form %}
{% if form %} {% if form %}

View file

@ -11,7 +11,7 @@ from django.template.defaultfilters import date
from django.utils.safestring import mark_safe from django.utils.safestring import mark_safe
from orchestra import get_version from orchestra import get_version
from orchestra.admin.utils import change_url from orchestra.admin.utils import change_url, admin_link as utils_admin_link
from orchestra.utils.apps import isinstalled from orchestra.utils.apps import isinstalled