From be00ab533c316fdf91454d6fe1ba46c31211254b Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Thu, 20 Nov 2014 16:48:50 +0000 Subject: [PATCH] Added support for resource monitoring on admin --- TODO.md | 9 --------- orchestra/admin/options.py | 8 ++++---- orchestra/apps/databases/admin.py | 4 ++-- orchestra/apps/mailboxes/admin.py | 4 ++-- orchestra/apps/resources/actions.py | 23 +++++++++++++++++------ orchestra/apps/resources/admin.py | 7 +++++-- orchestra/apps/resources/helpers.py | 2 +- orchestra/apps/resources/models.py | 12 ++++++++++-- orchestra/apps/resources/tasks.py | 4 ++-- orchestra/apps/webapps/admin.py | 2 +- orchestra/apps/websites/admin.py | 2 +- orchestra/apps/websites/settings.py | 2 +- 12 files changed, 46 insertions(+), 33 deletions(-) diff --git a/TODO.md b/TODO.md index cfb8ea22..15b3e077 100644 --- a/TODO.md +++ b/TODO.md @@ -182,19 +182,10 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when) - -* rename admin prefetch_related to list_prefetch_related for consistency - - -* LAST resource monitor option -> SUM(last backend) - * Resource.monitor(async=True) admin action * Validate a model path exists between resource.content_type and backend.model -* Add support for whitelisted IPs on traffic monitoring ['127.0.0.1',] - - * Periodic task for cleaning old monitoring data * Generate reports of Account contracted services diff --git a/orchestra/admin/options.py b/orchestra/admin/options.py index 78bec761..dba228cf 100644 --- a/orchestra/admin/options.py +++ b/orchestra/admin/options.py @@ -95,7 +95,7 @@ class ChangeViewActionsMixin(object): return views def change_view(self, request, object_id, **kwargs): - if not 'extra_context' in kwargs: + if kwargs.get('extra_context', None) is None: kwargs['extra_context'] = {} obj = self.get_object(request, unquote(object_id)) kwargs['extra_context']['object_tools_items'] = [ @@ -153,12 +153,12 @@ class ChangeAddFieldsMixin(object): class ExtendedModelAdmin(ChangeViewActionsMixin, ChangeAddFieldsMixin, admin.ModelAdmin): - prefetch_related = None + list_prefetch_related = None def get_queryset(self, request): qs = super(ExtendedModelAdmin, self).get_queryset(request) - if self.prefetch_related: - qs = qs.prefetch_related(*self.prefetch_related) + if self.list_prefetch_related: + qs = qs.prefetch_related(*self.list_prefetch_related) return qs diff --git a/orchestra/apps/databases/admin.py b/orchestra/apps/databases/admin.py index 8bbd05bd..f2b6303d 100644 --- a/orchestra/apps/databases/admin.py +++ b/orchestra/apps/databases/admin.py @@ -42,7 +42,7 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): readonly_fields = ('account_link', 'display_users',) filter_horizontal = ['users'] filter_by_account_fields = ('users',) - prefetch_related = ('users',) + list_prefetch_related = ('users',) def display_users(self, db): links = [] @@ -90,7 +90,7 @@ class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, Exten ) readonly_fields = ('account_link', 'display_databases',) filter_by_account_fields = ('databases',) - prefetch_related = ('databases',) + list_prefetch_related = ('databases',) def display_databases(self, user): links = [] diff --git a/orchestra/apps/mailboxes/admin.py b/orchestra/apps/mailboxes/admin.py index dc65f631..7571a0e0 100644 --- a/orchestra/apps/mailboxes/admin.py +++ b/orchestra/apps/mailboxes/admin.py @@ -60,7 +60,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo change_readonly_fields = ('name',) add_form = MailboxCreationForm form = MailboxChangeForm - prefetch_related = ('addresses__domain',) + list_prefetch_related = ('addresses__domain',) def display_addresses(self, mailbox): addresses = [] @@ -106,7 +106,7 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): filter_by_account_fields = ('domain', 'mailboxes') filter_horizontal = ['mailboxes'] form = AddressForm - prefetch_related = ('mailboxes', 'domain') + list_prefetch_related = ('mailboxes', 'domain') domain_link = admin_link('domain', order='domain__name') diff --git a/orchestra/apps/resources/actions.py b/orchestra/apps/resources/actions.py index 12668833..1e7532f1 100644 --- a/orchestra/apps/resources/actions.py +++ b/orchestra/apps/resources/actions.py @@ -4,18 +4,29 @@ from django.shortcuts import redirect from django.utils.translation import ungettext, ugettext_lazy as _ -@transaction.atomic def run_monitor(modeladmin, request, queryset): + """ Resource and ResourceData run monitors """ + referer = request.META.get('HTTP_REFERER') + if not queryset: + modeladmin.message_user(request, _("No resource has been selected,")) + return redirect(referer) for resource in queryset: resource.monitor() modeladmin.log_change(request, resource, _("Run monitors")) num = len(queryset) - msg = ungettext( - _("One selected resource has been monitored."), - _("%s selected resource have been monitored.") % num, - num) + async = resource.monitor.func_defaults[0] + if async: + # TODO schedulet link to celery taskstate page + msg = ungettext( + _("One selected resource has been scheduled for monitoring."), + _("%s selected resource have been scheduled for monitoring.") % num, + num) + else: + msg = ungettext( + _("One selected resource has been monitored."), + _("%s selected resource have been monitored.") % num, + num) modeladmin.message_user(request, msg) - referer = request.META.get('HTTP_REFERER') if referer: return redirect(referer) run_monitor.url_name = 'monitor' diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index f778d51d..ed050e0e 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -40,6 +40,8 @@ class ResourceAdmin(ExtendedModelAdmin): 'fields': ('monitors', 'crontab'), }), ) + actions = (run_monitor,) + change_view_actions = actions change_readonly_fields = ('name', 'content_type') prepopulated_fields = {'name': ('verbose_name',)} @@ -60,7 +62,8 @@ class ResourceAdmin(ExtendedModelAdmin): ) % { 'not_routed': ', '.join(not_routed) }) - return super(ResourceAdmin, self).changeform_view(request, object_id, form_url, extra_context) + return super(ResourceAdmin, self).change_view(request, object_id, form_url=form_url, + extra_context=extra_context) def save_model(self, request, obj, form, change): super(ResourceAdmin, self).save_model(request, obj, form, change) @@ -98,7 +101,7 @@ class ResourceDataAdmin(ExtendedModelAdmin): change_view_actions = actions ordering = ('-updated_at',) list_select_related = ('resource__content_type',) - prefetch_related = ('content_object',) + list_prefetch_related = ('content_object',) resource_link = admin_link('resource') content_object_link = admin_link('content_object') diff --git a/orchestra/apps/resources/helpers.py b/orchestra/apps/resources/helpers.py index f2c95de3..0131d18c 100644 --- a/orchestra/apps/resources/helpers.py +++ b/orchestra/apps/resources/helpers.py @@ -29,7 +29,7 @@ def compute_resource_usage(data): has_result = True result += sum(values) elif resource.period == resource.LAST: - result = dataset.value + result += dataset.value has_result = True else: raise NotImplementedError("%s support not implemented" % data.period) diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index f783e163..d1e2f433 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -126,6 +126,11 @@ class Resource(models.Model): def get_verbose_name(self): return self.verbose_name or self.name + + def monitor(self, async=True): + if async: + return tasks.monitor.delay(self.pk, async=async) + tasks.monitor(self.pk, async=async) class ResourceData(models.Model): @@ -174,8 +179,11 @@ class ResourceData(models.Model): self.updated_at = timezone.now() self.save(update_fields=['used', 'updated_at']) - def monitor(self): - tasks.monitor(self.resource_id, ids=(self.object_id,)) + def monitor(self, async=False): + ids = (self.object_id,) + if async: + return tasks.monitor.delay(self.resource_id, ids=ids, async=async) + return tasks.monitor(self.resource_id, ids=ids, async=async) def get_monitor_datasets(self): resource = self.resource diff --git a/orchestra/apps/resources/tasks.py b/orchestra/apps/resources/tasks.py index 2f1b2de5..3c4c9cd6 100644 --- a/orchestra/apps/resources/tasks.py +++ b/orchestra/apps/resources/tasks.py @@ -7,7 +7,7 @@ from .backends import ServiceMonitor @shared_task(name='resources.Monitor') -def monitor(resource_id, ids=None): +def monitor(resource_id, ids=None, async=True): from .models import ResourceData, Resource resource = Resource.objects.get(pk=resource_id) @@ -28,7 +28,7 @@ def monitor(resource_id, ids=None): for obj in model.objects.filter(**kwargs): operations.append(Operation.create(backend, obj, Operation.MONITOR)) # TODO async=TRue only when running with celery - Operation.execute(operations, async=True) + Operation.execute(operations, async=async) kwargs = {'id__in': ids} if ids else {} # Update used resources and trigger resource exceeded and revovery diff --git a/orchestra/apps/webapps/admin.py b/orchestra/apps/webapps/admin.py index 25797ace..a927a398 100644 --- a/orchestra/apps/webapps/admin.py +++ b/orchestra/apps/webapps/admin.py @@ -45,7 +45,7 @@ class WebAppAdmin(AccountAdminMixin, ExtendedModelAdmin): inlines = [WebAppOptionInline] readonly_fields = ('account_link',) change_readonly_fields = ('name', 'type') - prefetch_related = ('content_set__website',) + list_prefetch_related = ('content_set__website',) TYPE_HELP_TEXT = { k: str(unicode(v.get('help_text', ''))) diff --git a/orchestra/apps/websites/admin.py b/orchestra/apps/websites/admin.py index 5d4ea655..d486acc9 100644 --- a/orchestra/apps/websites/admin.py +++ b/orchestra/apps/websites/admin.py @@ -66,7 +66,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): }), ) filter_by_account_fields = ['domains'] - prefetch_related = ('domains', 'content_set__webapp') + list_prefetch_related = ('domains', 'content_set__webapp') search_fields = ('name', 'account__username', 'domains__name') def display_domains(self, website): diff --git a/orchestra/apps/websites/settings.py b/orchestra/apps/websites/settings.py index 46767432..78065ade 100644 --- a/orchestra/apps/websites/settings.py +++ b/orchestra/apps/websites/settings.py @@ -87,4 +87,4 @@ WEBSITES_WEBSITE_WWW_LOG_PATH = getattr(settings, 'WEBSITES_WEBSITE_WWW_LOG_PATH WEBSITES_TRAFFIC_IGNORE_HOSTS = getattr(settings, 'WEBSITES_TRAFFIC_IGNORE_HOSTS', - []) + ('127.0.0.1',))