From 86cfb10b9b255468bdde6d7d0476e457dba24467 Mon Sep 17 00:00:00 2001
From: Jens Langhammer <jens.langhammer@beryju.org>
Date: Fri, 16 Oct 2020 11:38:49 +0200
Subject: [PATCH] outposts: implement .run_wuth_logs() which returns logs, add
 task monitoring

---
 passbook/outposts/controllers/base.py |  9 +++++++-
 passbook/outposts/settings.py         |  2 +-
 passbook/outposts/tasks.py            | 30 +++++++++++++++++----------
 3 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/passbook/outposts/controllers/base.py b/passbook/outposts/controllers/base.py
index e7c9e250c..226937b88 100644
--- a/passbook/outposts/controllers/base.py
+++ b/passbook/outposts/controllers/base.py
@@ -1,7 +1,8 @@
 """Base Controller"""
-from typing import Dict
+from typing import Dict, List
 
 from structlog import get_logger
+from structlog.testing import capture_logs
 
 from passbook.lib.sentry import SentryIgnoredException
 from passbook.outposts.models import Outpost
@@ -29,6 +30,12 @@ class BaseController:
         """Called by scheduled task to reconcile deployment/service/etc"""
         raise NotImplementedError
 
+    def run_with_logs(self) -> List[str]:
+        """Call .run() but capture all log output and return it."""
+        with capture_logs() as logs:
+            self.run()
+        return logs
+
     def get_static_deployment(self) -> str:
         """Return a static deployment configuration"""
         raise NotImplementedError
diff --git a/passbook/outposts/settings.py b/passbook/outposts/settings.py
index 09e4a14fc..0a7d803a6 100644
--- a/passbook/outposts/settings.py
+++ b/passbook/outposts/settings.py
@@ -3,7 +3,7 @@ from celery.schedules import crontab
 
 CELERY_BEAT_SCHEDULE = {
     "outposts_controller": {
-        "task": "passbook.outposts.tasks.outpost_controller",
+        "task": "passbook.outposts.tasks.outpost_controller_all",
         "schedule": crontab(minute="*/5"),
         "options": {"queue": "passbook_scheduled"},
     },
diff --git a/passbook/outposts/tasks.py b/passbook/outposts/tasks.py
index 305759cae..5aafa2e10 100644
--- a/passbook/outposts/tasks.py
+++ b/passbook/outposts/tasks.py
@@ -6,7 +6,9 @@ from channels.layers import get_channel_layer
 from django.db.models.base import Model
 from structlog import get_logger
 
+from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
 from passbook.lib.utils.reflection import path_to_class
+from passbook.outposts.controllers.base import ControllerException
 from passbook.outposts.models import (
     Outpost,
     OutpostDeploymentType,
@@ -22,24 +24,30 @@ LOGGER = get_logger()
 
 
 @CELERY_APP.task()
-def outpost_controller():
+def outpost_controller_all():
     """Launch Controller for all Outposts which support it"""
     for outpost in Outpost.objects.exclude(
         deployment_type=OutpostDeploymentType.CUSTOM
     ):
-        outpost_controller_single.delay(
-            outpost.pk.hex, outpost.deployment_type, outpost.type
-        )
+        outpost_controller.delay(outpost.pk.hex, outpost.deployment_type, outpost.type)
 
 
-@CELERY_APP.task()
-def outpost_controller_single(outpost_pk: str, deployment_type: str, outpost_type: str):
+@CELERY_APP.task(bind=True, base=MonitoredTask)
+def outpost_controller(
+    self: MonitoredTask, outpost_pk: str, deployment_type: str, outpost_type: str
+):
     """Launch controller and reconcile deployment/service/etc"""
-    if outpost_type == OutpostType.PROXY:
-        if deployment_type == OutpostDeploymentType.KUBERNETES:
-            ProxyKubernetesController(outpost_pk).run()
-        if deployment_type == OutpostDeploymentType.DOCKER:
-            ProxyDockerController(outpost_pk).run()
+    logs = []
+    try:
+        if outpost_type == OutpostType.PROXY:
+            if deployment_type == OutpostDeploymentType.KUBERNETES:
+                logs = ProxyKubernetesController(outpost_pk).run_with_logs()
+            if deployment_type == OutpostDeploymentType.DOCKER:
+                logs = ProxyDockerController(outpost_pk).run_with_logs()
+    except ControllerException as exc:
+        self.set_status(TaskResult(TaskResultStatus.ERROR, [str(exc)], exc))
+    else:
+        self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL, logs))
 
 
 @CELERY_APP.task()