From 72ef63ffdf85f08349200fac6afb4b8d37acda8d Mon Sep 17 00:00:00 2001 From: Marc Date: Fri, 25 Jul 2014 15:17:50 +0000 Subject: [PATCH] Improved monitoring backends --- TODO.md | 2 +- orchestra/apps/lists/backends.py | 31 +++++-- orchestra/apps/orchestration/admin.py | 4 +- orchestra/apps/orchestration/backends.py | 4 + orchestra/apps/orchestration/manager.py | 2 +- orchestra/apps/orchestration/models.py | 2 +- orchestra/apps/resources/models.py | 5 +- orchestra/apps/resources/serializers.py | 4 +- orchestra/apps/users/backends.py | 102 +++++++++++---------- orchestra/apps/websites/backends/apache.py | 80 +++++++++------- orchestra/conf/base_settings.py | 4 +- 11 files changed, 139 insertions(+), 101 deletions(-) diff --git a/TODO.md b/TODO.md index 0cd52e1a..f04b9aa2 100644 --- a/TODO.md +++ b/TODO.md @@ -51,7 +51,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * create custom field that returns backend python objects * Timezone awareness on monitoring system (reading server-side logs with different TZ than orchestra) maybe a settings value? (use UTC internally, timezone.localtime() when interacting with servers) -* Resource metric: KB MB B? RESOURCE UNIT!! forms and serializers * EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0) @@ -72,3 +71,4 @@ at + clock time, midnight, noon- At 3:30 p.m., At 4:01, At noon * backend logs with hal logo +* Use logs for storing monitored values diff --git a/orchestra/apps/lists/backends.py b/orchestra/apps/lists/backends.py index 40239d8c..51644cc1 100644 --- a/orchestra/apps/lists/backends.py +++ b/orchestra/apps/lists/backends.py @@ -1,3 +1,5 @@ +import textwrap + from django.template import Template, Context from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -17,24 +19,35 @@ class MailmanTraffic(ServiceMonitor): model = 'lists.List' resource = ServiceMonitor.TRAFFIC + def prepare(self): + current_date = timezone.localtime(self.current_date) + current_date = current_date.strftime("%b %d %H:%M:%S") + self.append(textwrap.dedent(""" + function monitor () { + OBJECT_ID=$1 + LAST_DATE=$2 + LIST_NAME="$3" + MAILMAN_LOG="$4" + + SUBSCRIBERS=$(list_members ${LIST_NAME} | wc -l) + SIZE=$(grep ' post to ${LIST_NAME} ' "${MAILMAN_LOG}" \ + | awk '\"$LAST_DATE\"<=$0 && $0<=\"%s\"' \ + | sed 's/.*size=\([0-9]*\).*/\\1/' \ + | tr '\\n' '+' \ + | xargs -i echo {} ) + echo ${OBJECT_ID} $(( ${SIZE}*${SUBSCRIBERS} )) + }""" % current_date)) + def monitor(self, mail_list): context = self.get_context(mail_list) self.append( - "SUBSCRIBERS=$(list_members %(list_name)s | wc -l)\n" - "SIZE=$(grep ' post to %(list_name)s ' %(mailman_log)s \\\n" - " | awk '\"%(last_date)s\"<=$0 && $0<=\"%(current_date)s\"' \\\n" - " | sed 's/.*size=\([0-9]*\).*/\\1/' \\\n" - " | tr '\\n' '+' \\\n" - " | xargs -i echo {}0 )\n" - "echo %(object_id)s $(( ${SIZE}*${SUBSCRIBERS} ))" % context) + 'monitor %(object_id)i %(last_date)s "%(list_name)s" "%(log_file)s"' % context) def get_context(self, mail_list): last_date = timezone.localtime(self.get_last_date(mail_list.pk)) - current_date = timezone.localtime(self.current_date) return { 'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH, 'list_name': mail_list.name, 'object_id': mail_list.pk, 'last_date': last_date.strftime("%b %d %H:%M:%S"), - 'current_date': current_date.strftime("%b %d %H:%M:%S"), } diff --git a/orchestra/apps/orchestration/admin.py b/orchestra/apps/orchestration/admin.py index cff38be3..283e8ce1 100644 --- a/orchestra/apps/orchestration/admin.py +++ b/orchestra/apps/orchestration/admin.py @@ -81,8 +81,8 @@ class BackendLogAdmin(admin.ModelAdmin): date_hierarchy = 'last_update' inlines = [BackendOperationInline] fields = [ - 'backend', 'server', 'state', 'mono_script', 'mono_stdout', 'mono_stderr', - 'mono_traceback', 'exit_code', 'task_id', 'display_created', + 'backend', 'server_link', 'state', 'mono_script', 'mono_stdout', + 'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created', 'display_last_update', 'execution_time' ] readonly_fields = fields diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index bb058d78..7f351d89 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -97,6 +97,10 @@ class ServiceBackend(plugins.Plugin): else: self.cmds[-1][1].append(cmd) + def prepare(self): + """ hook for executing something at the beging """ + pass + def commit(self): """ apply the configuration, usually reloading a service diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index ce9360f8..da166150 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -38,11 +38,11 @@ def execute(operations): cache = {} for operation in operations: servers = router.get_servers(operation, cache=cache) - print cache for server in servers: key = (server, operation.backend) if key not in scripts: scripts[key] = (operation.backend(), [operation]) + scripts[key][0].prepare() else: scripts[key][1].append(operation) method = getattr(scripts[key][0], operation.action) diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index bdafe42e..e5ea04a7 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -60,7 +60,7 @@ class BackendLog(models.Model): traceback = models.TextField(_("traceback")) exit_code = models.IntegerField(_("exit code"), null=True) task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True, - help_text="Celery task ID") + help_text="Celery task ID when used as execution backend") created = models.DateTimeField(_("created"), auto_now_add=True) last_update = models.DateTimeField(_("last update"), auto_now=True) diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index a91d37ae..2e674198 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -129,10 +129,10 @@ class ResourceData(models.Model): ct = ContentType.objects.get_for_model(type(obj)) try: return cls.objects.get(content_type=ct, object_id=obj.pk, - resource=resource) + resource=resource) except cls.DoesNotExist: return cls.objects.create(content_object=obj, resource=resource, - allocated=resource.default_allocation) + allocated=resource.default_allocation) def get_used(self): return helpers.compute_resource_usage(self) @@ -173,6 +173,7 @@ def create_resource_relation(): return data def __get__(self, obj, cls): + """ proxy handled object """ self.obj = obj return self diff --git a/orchestra/apps/resources/serializers.py b/orchestra/apps/resources/serializers.py index 7be4393c..b2d57b87 100644 --- a/orchestra/apps/resources/serializers.py +++ b/orchestra/apps/resources/serializers.py @@ -25,8 +25,8 @@ class ResourceSerializer(serializers.ModelSerializer): if not running_syncdb(): # TODO why this is even loaded during syncdb? - for resources in Resource.group_by_content_type(): - model = resources[0].content_type.model_class() + for ct, resources in Resource.objects.group_by('content_type'): + model = ct.model_class() try: router.insert(model, 'resources', ResourceSerializer, required=False, many=True) except KeyError: diff --git a/orchestra/apps/users/backends.py b/orchestra/apps/users/backends.py index c209c9f4..b273243c 100644 --- a/orchestra/apps/users/backends.py +++ b/orchestra/apps/users/backends.py @@ -1,3 +1,5 @@ +import textwrap + from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -15,16 +17,16 @@ class SystemUserBackend(ServiceController): def save(self, user): context = self.get_context(user) if user.is_main: - self.append( - "if [[ $( id %(username)s ) ]]; then \n" - " usermod --password '%(password)s' %(username)s \n" - "else \n" - " useradd %(username)s --password '%(password)s' \\\n" - " --shell /bin/false \n" - "fi" % context - ) - self.append("mkdir -p %(home)s" % context) - self.append("chown %(username)s.%(username)s %(home)s" % context) + self.append(textwrap.dedent(""" + if [[ $( id %(username)s ) ]]; then + usermod --password '%(password)s' %(username)s + else + useradd %(username)s --password '%(password)s' \ + --shell /bin/false + fi + mkdir -p %(home)s + chown %(username)s.%(username)s %(home)s""" % context + )) else: self.delete(user) @@ -63,50 +65,58 @@ class FTPTraffic(ServiceMonitor): resource = ServiceMonitor.TRAFFIC verbose_name = _('FTP traffic') + def prepare(self): + current_date = timezone.localtime(self.current_date) + current_date = current_date.strftime("%Y%m%d%H%M%S") + self.append(textwrap.dedent(""" + function monitor () { + OBJECT_ID=$1 + INI_DATE=$2 + USERNAME="$3" + LOG_FILE="$4" + grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \ + | grep " \\[${USERNAME}\\] " \ + | awk -v ini="${INI_DATE}" ' + BEGIN { + end = "%s" + sum = 0 + months["Jan"] = "01" + months["Feb"] = "02" + months["Mar"] = "03" + months["Apr"] = "04" + months["May"] = "05" + months["Jun"] = "06" + months["Jul"] = "07" + months["Aug"] = "08" + months["Sep"] = "09" + months["Oct"] = "10" + months["Nov"] = "11" + months["Dec"] = "12" + } { + # log: Fri Jul 11 13:23:17 2014 + split($4, t, ":") + # line_date = year month day hour minute second + line_date = $5 months[$2] $3 t[1] t[2] t[3] + if ( line_date > ini && line_date < end) + split($0, l, "\\", ") + split(l[3], b, " ") + sum += b[1] + } END { + print sum + } + ' | xargs echo ${OBJECT_ID} + }""" % current_date)) + def monitor(self, user): context = self.get_context(user) - self.append(""" - grep "UPLOAD\|DOWNLOAD" %(log_file)s | grep " \\[%(username)s\\] " | awk ' - BEGIN { - ini = "%(last_date)s" - end = "%(current_date)s" - - months["Jan"] = "01" - months["Feb"] = "02" - months["Mar"] = "03" - months["Apr"] = "04" - months["May"] = "05" - months["Jun"] = "06" - months["Jul"] = "07" - months["Aug"] = "08" - months["Sep"] = "09" - months["Oct"] = "10" - months["Nov"] = "11" - months["Dec"] = "12" - } { - # log: Fri Jul 11 13:23:17 2014 - split($4, t, ":") - # line_date = year month day hour minute second - line_date = $5 months[$2] $3 t[1] t[2] t[3] - if ( line_date > ini && line_date < end) - split($0, l, "\\", ") - split(l[3], b, " ") - sum += b[1] - } END { - if ( sum ) - print sum - else - print 0 - } - ' | xargs echo %(object_id)s """ % context) + self.append( + 'monitor %(object_id)i %(last_date)s "%(username)s" "%(log_file)s"' % context) def get_context(self, user): last_date = timezone.localtime(self.get_last_date(user.pk)) - current_date = timezone.localtime(self.current_date) return { 'log_file': settings.USERS_FTP_LOG_PATH, 'last_date': last_date.strftime("%Y%m%d%H%M%S"), - 'current_date': current_date.strftime("%Y%m%d%H%M%S"), 'object_id': user.pk, 'username': user.username, } diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index a58ce5f2..57c337e6 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -1,3 +1,4 @@ +import textwrap import os from django.template import Template, Context @@ -182,48 +183,57 @@ class Apache2Traffic(ServiceMonitor): resource = ServiceMonitor.TRAFFIC verbose_name = _("Apache 2 Traffic") + def prepare(self): + current_date = timezone.localtime(self.current_date) + current_date = current_date.strftime("%Y%m%d%H%M%S") + self.append(textwrap.dedent(""" + function monitor () { + OBJECT_ID=$1 + INI_DATE=$2 + LOG_FILE="$3" + { + awk -v ini="${INI_DATE}" ' + BEGIN { + end = "%s" + sum = 0 + months["Jan"] = "01"; + months["Feb"] = "02"; + months["Mar"] = "03"; + months["Apr"] = "04"; + months["May"] = "05"; + months["Jun"] = "06"; + months["Jul"] = "07"; + months["Aug"] = "08"; + months["Sep"] = "09"; + months["Oct"] = "10"; + months["Nov"] = "11"; + months["Dec"] = "12"; + } { + # date = [11/Jul/2014:13:50:41 + date = substr($4, 2) + year = substr(date, 8, 4) + month = months[substr(date, 4, 3)]; + day = substr(date, 1, 2) + hour = substr(date, 13, 2) + minute = substr(date, 16, 2) + second = substr(date, 19, 2) + line_date = year month day hour minute second + if ( line_date > ini && line_date < end) + sum += $NF + } END { + print sum + }' "${LOG_FILE}" || echo 0 + } | xargs echo ${OBJECT_ID} + }""" % current_date)) + def monitor(self, site): context = self.get_context(site) - self.append("""{ - awk 'BEGIN { - ini = "%(last_date)s" - end = "%(current_date)s" - sum = 0 - - months["Jan"] = "01"; - months["Feb"] = "02"; - months["Mar"] = "03"; - months["Apr"] = "04"; - months["May"] = "05"; - months["Jun"] = "06"; - months["Jul"] = "07"; - months["Aug"] = "08"; - months["Sep"] = "09"; - months["Oct"] = "10"; - months["Nov"] = "11"; - months["Dec"] = "12"; - } { - # date = [11/Jul/2014:13:50:41 - date = substr($4, 2) - year = substr(date, 8, 4) - month = months[substr(date, 4, 3)]; - day = substr(date, 1, 2) - hour = substr(date, 13, 2) - minute = substr(date, 16, 2) - second = substr(date, 19, 2) - line_date = year month day hour minute second - if ( line_date > ini && line_date < end) - sum += $NF - } END { - print sum - }' %(log_file)s || echo 0; } | xargs echo %(object_id)s """ % context) + self.append('monitor %(object_id)i %(last_date)s "%(log_file)s"' % context) def get_context(self, site): last_date = timezone.localtime(self.get_last_date(site.pk)) - current_date = timezone.localtime(self.current_date) return { 'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name), 'last_date': last_date.strftime("%Y%m%d%H%M%S"), - 'current_date': current_date.strftime("%Y%m%d%H%M%S"), 'object_id': site.pk, } diff --git a/orchestra/conf/base_settings.py b/orchestra/conf/base_settings.py index 57b1f41f..3f99ada0 100644 --- a/orchestra/conf/base_settings.py +++ b/orchestra/conf/base_settings.py @@ -118,8 +118,8 @@ AUTHENTICATION_BACKENDS = [ ] -# Email config -EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend' +#TODO Email config +#EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend' #################################