diff --git a/lifecycle/bootstrap.sh b/lifecycle/bootstrap.sh index 46cc932fa..f36e44e37 100755 --- a/lifecycle/bootstrap.sh +++ b/lifecycle/bootstrap.sh @@ -4,7 +4,7 @@ printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", if [[ "$1" == "server" ]]; then gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application elif [[ "$1" == "worker" ]]; then - celery -A passbook.root.celery worker --autoscale 10,3 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled + celery -A passbook.root.celery worker --autoscale 3,1 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled elif [[ "$1" == "migrate" ]]; then # Run system migrations first, run normal migrations after python -m lifecycle.migrate diff --git a/passbook/admin/tasks.py b/passbook/admin/tasks.py index 931be54ea..a0551eb3a 100644 --- a/passbook/admin/tasks.py +++ b/passbook/admin/tasks.py @@ -3,6 +3,7 @@ from django.core.cache import cache from requests import RequestException, get from structlog import get_logger +from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus from passbook.root.celery import CELERY_APP LOGGER = get_logger() @@ -10,8 +11,8 @@ VERSION_CACHE_KEY = "passbook_latest_version" VERSION_CACHE_TIMEOUT = 2 * 60 * 60 # 2 hours -@CELERY_APP.task() -def update_latest_version(): +@CELERY_APP.task(bind=True, base=MonitoredTask) +def update_latest_version(self: MonitoredTask): """Update latest version info""" try: data = get( @@ -19,5 +20,11 @@ def update_latest_version(): ).json() tag_name = data.get("tag_name") cache.set(VERSION_CACHE_KEY, tag_name.split("/")[1], VERSION_CACHE_TIMEOUT) - except (RequestException, IndexError): + self.set_status( + TaskResult( + TaskResultStatus.SUCCESSFUL, ["Successfully updated latest Version"] + ) + ) + except (RequestException, IndexError) as exc: cache.set(VERSION_CACHE_KEY, "0.0.0", VERSION_CACHE_TIMEOUT) + self.set_status(TaskResult(TaskResultStatus.ERROR, [str(exc)])) diff --git a/passbook/admin/templates/administration/base.html b/passbook/admin/templates/administration/base.html index c5213a97f..ba4ce6e9f 100644 --- a/passbook/admin/templates/administration/base.html +++ b/passbook/admin/templates/administration/base.html @@ -146,6 +146,12 @@ {% trans 'Groups' %} +
  • + + {% trans 'System Tasks' %} + +
    • diff --git a/passbook/admin/templates/administration/task/list.html b/passbook/admin/templates/administration/task/list.html new file mode 100644 index 000000000..716aed06a --- /dev/null +++ b/passbook/admin/templates/administration/task/list.html @@ -0,0 +1,71 @@ +{% extends "administration/base.html" %} + +{% load i18n %} +{% load humanize %} +{% load passbook_utils %} + +{% block content %} +
    +
    +

    + + {% trans 'System Tasks' %} +

    +

    {% trans "Background tasks." %}

    +
    +
    +
    +
    + + + + + + + + + + + + {% for key, task in object_list.items %} + + + + + + + + {% endfor %} + +
    {% trans 'Identifier' %}{% trans 'Description' %}{% trans 'Last Status' %}{% trans 'Status' %}{% trans 'Messages' %}
    + 
    {{ key }}
    +     		+ + {{ task.task_description }} + + + + {{ task.finish_timestamp|naturaltime }} + + + + {% if task.result.status == task_successful %} + {% trans 'Successful' %} + {% elif task.result.status == task_warning %} + {% trans 'Warning' %} + {% elif task.result.status == task_error %} + {% trans 'Error' %} + {% else %} + {% trans 'Unknown' %} + {% endif %} + + + {% for message in task.result.messages %} +
    + {{ message }} +
    + {% endfor %} +
    +
    +
    +{% endblock %} diff --git a/passbook/admin/urls.py b/passbook/admin/urls.py index 86a52ca40..e2c13505e 100644 --- a/passbook/admin/urls.py +++ b/passbook/admin/urls.py @@ -17,6 +17,7 @@ from passbook.admin.views import ( stages_bindings, stages_invitations, stages_prompts, + tasks, tokens, users, ) @@ -311,4 +312,10 @@ urlpatterns = [ outposts.OutpostDeleteView.as_view(), name="outpost-delete", ), + # Tasks + path( + "tasks/", + tasks.TaskListView.as_view(), + name="tasks", + ), ] diff --git a/passbook/admin/views/tasks.py b/passbook/admin/views/tasks.py new file mode 100644 index 000000000..dc8cc7586 --- /dev/null +++ b/passbook/admin/views/tasks.py @@ -0,0 +1,22 @@ +"""passbook Tasks List""" +from typing import Any, Dict + +from django.core.cache import cache +from django.views.generic.base import TemplateView + +from passbook.admin.mixins import AdminRequiredMixin +from passbook.lib.tasks import TaskResultStatus + + +class TaskListView(AdminRequiredMixin, TemplateView): + """Show list of all background tasks""" + + template_name = "administration/task/list.html" + + def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: + kwargs = super().get_context_data(**kwargs) + kwargs["object_list"] = cache.get_many(cache.keys("task_*")) + kwargs["task_successful"] = TaskResultStatus.SUCCESSFUL + kwargs["task_warning"] = TaskResultStatus.WARNING + kwargs["task_error"] = TaskResultStatus.ERROR + return kwargs diff --git a/passbook/core/tasks.py b/passbook/core/tasks.py index 16470291e..2822458d4 100644 --- a/passbook/core/tasks.py +++ b/passbook/core/tasks.py @@ -3,14 +3,16 @@ from django.utils.timezone import now from structlog import get_logger from passbook.core.models import ExpiringModel +from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus from passbook.root.celery import CELERY_APP LOGGER = get_logger() -@CELERY_APP.task() -def clean_expired_models(): +@CELERY_APP.task(bind=True, base=MonitoredTask) +def clean_expired_models(self: MonitoredTask): """Remove expired objects""" + messages = [] for cls in ExpiringModel.__subclasses__(): cls: ExpiringModel amount, _ = ( @@ -20,3 +22,5 @@ def clean_expired_models(): .delete() ) LOGGER.debug("Deleted expired models", model=cls, amount=amount) + messages.append(f"Deleted {amount} expired {cls._meta.verbose_name_plural}") + self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL, messages)) diff --git a/passbook/core/tests/test_tasks.py b/passbook/core/tests/test_tasks.py index f3144f313..90dd1c2aa 100644 --- a/passbook/core/tests/test_tasks.py +++ b/passbook/core/tests/test_tasks.py @@ -14,5 +14,5 @@ class TestTasks(TestCase): """Test Token cleanup task""" Token.objects.create(expires=now(), user=get_anonymous_user()) self.assertEqual(Token.objects.all().count(), 1) - clean_expired_models() + clean_expired_models.delay() self.assertEqual(Token.objects.all().count(), 0) diff --git a/passbook/lib/tasks.py b/passbook/lib/tasks.py new file mode 100644 index 000000000..bc95f320c --- /dev/null +++ b/passbook/lib/tasks.py @@ -0,0 +1,88 @@ +"""Monitored tasks""" +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import List, Optional + +from celery import Task +from django.core.cache import cache + + +class TaskResultStatus(Enum): + """Possible states of tasks""" + + SUCCESSFUL = 1 + WARNING = 2 + ERROR = 4 + + +@dataclass +class TaskResult: + """Result of a task run, this class is created by the task itself + and used by self.set_status""" + + status: TaskResultStatus + + messages: List[str] = field(default_factory=list) + + error: Optional[Exception] = field(default=None) + + # Optional UID used in cache for tasks that run in different instances + uid: Optional[str] = field(default=None) + + +@dataclass +class TaskInfo: + """Info about a task run""" + + task_name: str + finish_timestamp: datetime + + result: TaskResult + + task_description: Optional[str] = field(default=None) + + def save(self): + """Save task into cache""" + key = f"task_{self.task_name}" + if self.result.uid: + key += f"_{self.result.uid}" + self.task_name += f"_{self.result.uid}" + cache.set(key, self) + + +class MonitoredTask(Task): + """Task which can save its state to the cache""" + + _result: TaskResult + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._result = TaskResult(status=TaskResultStatus.ERROR, messages=[]) + + def set_status(self, result: TaskResult): + """Set result for current run, will overwrite previous result.""" + self._result = result + + # pylint: disable=too-many-arguments + def after_return(self, status, retval, task_id, args, kwargs, einfo): + TaskInfo( + task_name=self.__name__, + task_description=self.__doc__, + finish_timestamp=datetime.now(), + result=self._result, + ).save() + return super().after_return(status, retval, task_id, args, kwargs, einfo=einfo) + + # pylint: disable=too-many-arguments + def on_failure(self, exc, task_id, args, kwargs, einfo): + TaskInfo( + task_name=self.__name__, + task_description=self.__doc__, + finish_timestamp=datetime.now(), + result=self._result, + ).save() + return super().on_failure(exc, task_id, args, kwargs, einfo=einfo) + + def run(self, *args, **kwargs): + raise NotImplementedError diff --git a/passbook/providers/proxy/tests.py b/passbook/providers/proxy/tests.py index 810986e0a..23d0a5aad 100644 --- a/passbook/providers/proxy/tests.py +++ b/passbook/providers/proxy/tests.py @@ -7,9 +7,7 @@ from django.test import TestCase from passbook.flows.models import Flow from passbook.outposts.models import Outpost, OutpostDeploymentType, OutpostType -from passbook.providers.proxy.controllers.kubernetes import ( - ProxyKubernetesController, -) +from passbook.providers.proxy.controllers.kubernetes import ProxyKubernetesController from passbook.providers.proxy.models import ProxyProvider diff --git a/passbook/stages/consent/tests.py b/passbook/stages/consent/tests.py index d262dab1a..893e9a1bc 100644 --- a/passbook/stages/consent/tests.py +++ b/passbook/stages/consent/tests.py @@ -127,7 +127,7 @@ class TestConsentStage(TestCase): ).exists() ) sleep(1) - clean_expired_models() + clean_expired_models.delay() self.assertFalse( UserConsent.objects.filter( user=self.user, application=self.application diff --git a/passbook/static/static/passbook/passbook.css b/passbook/static/static/passbook/passbook.css index 645e7d314..d0d5e1997 100644 --- a/passbook/static/static/passbook/passbook.css +++ b/passbook/static/static/passbook/passbook.css @@ -52,6 +52,9 @@ .pf-m-success { color: var(--pf-global--success-color--100); } +.pf-m-warning { + color: var(--pf-global--warning-color--100); +} .pf-m-danger { color: var(--pf-global--danger-color--100); }