outposts: save state of outposts

passbook/outposts/controllers/base.py

@@ -9,7 +9,7 @@ from passbook.outposts.models import Outpost, OutpostServiceConnection
 
 
 class ControllerException(SentryIgnoredException):
-    """Exception raise when anything fails during controller run"""
+    """Exception raised when anything fails during controller run"""
 
 
 class BaseController:

passbook/outposts/controllers/docker.py

@@ -10,7 +10,11 @@ from yaml import safe_dump
 
 from passbook import __version__
 from passbook.outposts.controllers.base import BaseController, ControllerException
-from passbook.outposts.models import DockerServiceConnection, Outpost
+from passbook.outposts.models import (
+    DockerServiceConnection,
+    Outpost,
+    ServiceConnectionInvalid,
+)
 
 
 class DockerController(BaseController):
@@ -26,14 +30,8 @@ class DockerController(BaseController):
     def __init__(self, outpost: Outpost, connection: DockerServiceConnection) -> None:
         super().__init__(outpost, connection)
         try:
-            if self.connection.local:
+            self.client = connection.client()
-                self.client = DockerClient.from_env()
+        except ServiceConnectionInvalid as exc:
-            else:
-                self.client = DockerClient(
-                    base_url=self.connection.url,
-                    tls=self.connection.tls,
-                )
-        except DockerException as exc:
             raise ControllerException from exc
 
     def _get_labels(self) -> Dict[str, str]:

passbook/outposts/controllers/kubernetes.py

@@ -3,9 +3,7 @@ from io import StringIO
 from typing import Dict, List, Type
 
 from kubernetes.client import OpenApiException
-from kubernetes.config import load_incluster_config, load_kube_config
+from kubernetes.client.api_client import ApiClient
-from kubernetes.config.config_exception import ConfigException
-from kubernetes.config.kube_config import load_kube_config_from_dict
 from structlog.testing import capture_logs
 from yaml import dump_all
 
@@ -23,19 +21,14 @@ class KubernetesController(BaseController):
     reconcilers: Dict[str, Type[KubernetesObjectReconciler]]
     reconcile_order: List[str]
 
+    config: ApiClient
     connection: KubernetesServiceConnection
 
     def __init__(
         self, outpost: Outpost, connection: KubernetesServiceConnection
     ) -> None:
         super().__init__(outpost, connection)
-        try:
+        self.client = connection.client()
-            if self.connection.local:
-                load_incluster_config()
-            else:
-                load_kube_config_from_dict(self.connection.kubeconfig)
-        except ConfigException:
-            load_kube_config()
         self.reconcilers = {
             "secret": SecretReconciler,
             "deployment": DeploymentReconciler,

passbook/outposts/migrations/0010_service_connection.py

@@ -4,9 +4,10 @@ import uuid
 
 import django.db.models.deletion
 from django.apps.registry import Apps
+from django.core.exceptions import FieldError
 from django.db import migrations, models
 from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.core.exceptions import FieldError
+
 import passbook.lib.models
 
 

passbook/outposts/models.py

@@ -5,14 +5,24 @@ from typing import Dict, Iterable, List, Optional, Type, Union
 from uuid import uuid4
 
 from dacite import from_dict
+from django.conf import settings
 from django.core.cache import cache
 from django.db import models, transaction
 from django.db.models.base import Model
 from django.forms.models import ModelForm
 from django.http import HttpRequest
 from django.utils.translation import gettext_lazy as _
+from docker.client import DockerClient
+from docker.errors import DockerException
 from guardian.models import UserObjectPermission
 from guardian.shortcuts import assign_perm
+from kubernetes.client import VersionApi, VersionInfo
+from kubernetes.client.api_client import ApiClient
+from kubernetes.client.configuration import Configuration
+from kubernetes.client.exceptions import OpenApiException
+from kubernetes.config.config_exception import ConfigException
+from kubernetes.config.incluster_config import load_incluster_config
+from kubernetes.config.kube_config import load_kube_config, load_kube_config_from_dict
 from model_utils.managers import InheritanceManager
 from packaging.version import LegacyVersion, Version, parse
 
@@ -20,12 +30,17 @@ from passbook import __version__
 from passbook.core.models import Provider, Token, TokenIntents, User
 from passbook.lib.config import CONFIG
 from passbook.lib.models import InheritanceForeignKey
+from passbook.lib.sentry import SentryIgnoredException
 from passbook.lib.utils.template import render_to_string
 
 OUR_VERSION = parse(__version__)
 OUTPOST_HELLO_INTERVAL = 10
 
 
+class ServiceConnectionInvalid(SentryIgnoredException):
+    """"Exception raised when a Service Connection has invalid parameters"""
+
+
 @dataclass
 class OutpostConfig:
     """Configuration an outpost uses to configure it self"""
@@ -68,6 +83,14 @@ def default_outpost_config():
     return asdict(OutpostConfig(passbook_host=""))
 
 
+@dataclass
+class OutpostServiceConnectionState:
+    """State of an Outpost Service Connection"""
+
+    version: str
+    healthy: bool
+
+
 class OutpostServiceConnection(models.Model):
     """Connection details for an Outpost Controller, like Docker or Kubernetes"""
 
@@ -87,6 +110,19 @@ class OutpostServiceConnection(models.Model):
 
     objects = InheritanceManager()
 
+    @property
+    def state(self) -> OutpostServiceConnectionState:
+        """Get state of service connection"""
+        state_key = f"outpost_service_connection_{self.pk.hex}"
+        state = cache.get(state_key, None)
+        if state:
+            state = self._get_state()
+            cache.set(state_key, state)
+        return state
+
+    def _get_state(self) -> OutpostServiceConnectionState:
+        raise NotImplementedError
+
     @property
     def form(self) -> Type[ModelForm]:
         """Return Form class used to edit this object"""
@@ -113,6 +149,31 @@ class DockerServiceConnection(OutpostServiceConnection):
     def __str__(self) -> str:
         return f"Docker Service-Connection {self.name}"
 
+    def client(self) -> DockerClient:
+        """Get DockerClient"""
+        try:
+            client = None
+            if self.local:
+                client = DockerClient.from_env()
+            else:
+                client = DockerClient(
+                    base_url=self.url,
+                    tls=self.tls,
+                )
+            client.containers.list()
+        except DockerException as exc:
+            raise ServiceConnectionInvalid from exc
+        return client
+
+    def _get_state(self) -> OutpostServiceConnectionState:
+        try:
+            client = self.client()
+            return OutpostServiceConnectionState(
+                version=client.info()["ServerVersion"], healthy=True
+            )
+        except ServiceConnectionInvalid:
+            return OutpostServiceConnectionState(version="", healthy=False)
+
     class Meta:
 
         verbose_name = _("Docker Service-Connection")
@@ -140,6 +201,32 @@ class KubernetesServiceConnection(OutpostServiceConnection):
     def __str__(self) -> str:
         return f"Kubernetes Service-Connection {self.name}"
 
+    def _get_state(self) -> OutpostServiceConnectionState:
+        try:
+            client = self.client()
+            api_instance = VersionApi(client)
+            version: VersionInfo = api_instance.get_code()
+            return OutpostServiceConnectionState(
+                version=version.git_version, healthy=True
+            )
+        except OpenApiException:
+            return OutpostServiceConnectionState(version="", healthy=False)
+
+    def client(self) -> ApiClient:
+        """Get Kubernetes client configured from kubeconfig"""
+        config = Configuration()
+        try:
+            if self.local:
+                load_incluster_config(client_configuration=config)
+            else:
+                load_kube_config_from_dict(self.kubeconfig, client_configuration=config)
+            return ApiClient(config)
+        except ConfigException as exc:
+            if not settings.DEBUG:
+                raise ServiceConnectionInvalid from exc
+            load_kube_config(client_configuration=config)
+            return config
+
     class Meta:
 
         verbose_name = _("Kubernetes Service-Connection")