outposts: improve controller error handling

passbook/outposts/controllers/base.py

@@ -3,9 +3,14 @@ from typing import Dict
 
 from structlog import get_logger
 
+from passbook.lib.sentry import SentryIgnoredException
 from passbook.outposts.models import Outpost
 
 
+class ControllerException(SentryIgnoredException):
+    """Exception raise when anything fails during controller run"""
+
+
 class BaseController:
     """Base Outpost deployment controller"""
 

passbook/outposts/controllers/docker.py

@@ -2,12 +2,12 @@
 from typing import Dict, Tuple
 
 from docker import DockerClient, from_env
-from docker.errors import NotFound
+from docker.errors import DockerException, NotFound
 from docker.models.containers import Container
 from yaml import safe_dump
 
 from passbook import __version__
-from passbook.outposts.controllers.base import BaseController
+from passbook.outposts.controllers.base import BaseController, ControllerException
 
 
 class DockerController(BaseController):
@@ -62,6 +62,7 @@ class DockerController(BaseController):
             )
 
     def run(self):
+        try:
             container, has_been_created = self._get_container()
             if has_been_created:
                 return None
@@ -99,6 +100,8 @@ class DockerController(BaseController):
                 self.logger.info("Container is not running, restarting...")
                 container.start()
             return None
+        except DockerException as exc:
+            raise ControllerException from exc
 
     def get_static_deployment(self) -> str:
         """Generate docker-compose yaml for proxy, version 3.5"""

passbook/outposts/controllers/kubernetes.py

@@ -1,11 +1,12 @@
 """Kubernetes deployment controller"""
 from io import StringIO
 
+from kubernetes.client import OpenApiException
 from kubernetes.config import load_incluster_config, load_kube_config
 from kubernetes.config.config_exception import ConfigException
 from yaml import dump_all
 
-from passbook.outposts.controllers.base import BaseController
+from passbook.outposts.controllers.base import BaseController, ControllerException
 from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler
 from passbook.outposts.controllers.k8s.secret import SecretReconciler
 from passbook.outposts.controllers.k8s.service import ServiceReconciler
@@ -23,6 +24,7 @@ class KubernetesController(BaseController):
 
     def run(self):
         """Called by scheduled task to reconcile deployment/service/etc"""
+        try:
             namespace = self.outpost.config.kubernetes_namespace
 
             secret_reconciler = SecretReconciler(self.outpost)
@@ -38,6 +40,8 @@ class KubernetesController(BaseController):
             service_reconciler.namespace = namespace
             service_reconciler.deployment_ports = self.deployment_ports
             service_reconciler.run()
+        except OpenApiException as exc:
+            raise ControllerException from exc
 
     def get_static_deployment(self) -> str:
         secret_reconciler = SecretReconciler(self.outpost)