tests: move integration tests into separate folder, add separate pipeline task

Makefile

@@ -1,19 +1,26 @@
 all: lint-fix lint coverage gen
 
+test-integration:
+	k3d cluster create || exit 0
+	k3d kubeconfig write -o ~/.kube/config --overwrite
+	coverage run manage.py test --failfast -v 3 tests/integration
+
+test-e2e:
+	coverage run manage.py test --failfast -v 3 tests/e2e
+
 coverage:
-	coverage run --concurrency=multiprocessing manage.py test --failfast -v 3
-	coverage combine
+	coverage run manage.py test --failfast -v 3 passbook
 	coverage html
 	coverage report
 
 lint-fix:
 	isort -rc .
-	black passbook e2e lifecycle
+	black passbook tests lifecycle
 
 lint:
-	pyright passbook e2e lifecycle
-	bandit -r passbook e2e lifecycle -x node_modules
-	pylint passbook e2e lifecycle
+	pyright passbook tests lifecycle
+	bandit -r passbook tests lifecycle -x node_modules
+	pylint passbook tests lifecycle
 	prospector
 
 gen: coverage

azure-pipelines.yml

@@ -31,7 +31,7 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run pylint passbook e2e lifecycle
+              script: pipenv run pylint passbook tests lifecycle
       - job: black
         pool:
           vmImage: 'ubuntu-latest'
@@ -47,7 +47,7 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run black --check passbook e2e lifecycle
+              script: pipenv run black --check passbook tests lifecycle
       - job: prospector
         pool:
           vmImage: 'ubuntu-latest'
@@ -80,7 +80,7 @@ stages:
                 pipenv install --dev
           - task: CmdLine@2
             inputs:
-              script: pipenv run bandit -r passbook e2e lifecycle
+              script: pipenv run bandit -r passbook tests lifecycle
       - job: pyright
         pool:
           vmImage: ubuntu-latest
@@ -179,13 +179,6 @@ stages:
               dockerComposeFile: 'scripts/ci.docker-compose.yml'
               action: 'Run services'
               buildImages: false
-          - task: CmdLine@2
-            displayName: Install K3d and prepare
-            inputs:
-              script: |
-                wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
-                k3d cluster create
-                k3d kubeconfig write -o ~/.kube/config --overwrite
           - task: CmdLine@2
             inputs:
               script: |
@@ -196,8 +189,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
-                export PB_TEST_K8S=true
-                pipenv run coverage run ./manage.py test passbook -v 3
+                pipenv run make coverage
           - task: CmdLine@2
             inputs:
               script: |
@@ -209,6 +201,48 @@ stages:
               targetPath: 'output-unittest/'
               artifact: 'coverage-unittest'
               publishLocation: 'pipeline'
+      - job: coverage_integration
+        pool:
+          vmImage: 'ubuntu-latest'
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.9'
+          - task: DockerCompose@0
+            displayName: Run services
+            inputs:
+              dockerComposeFile: 'scripts/ci.docker-compose.yml'
+              action: 'Run services'
+              buildImages: false
+          - task: CmdLine@2
+            inputs:
+              script: |
+                sudo apt install -y libxmlsec1-dev pkg-config
+                sudo pip install -U wheel pipenv
+                pipenv install --dev
+          - task: CmdLine@2
+            displayName: Install K3d and prepare
+            inputs:
+              script: |
+                wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+                k3d cluster create
+                k3d kubeconfig write -o ~/.kube/config --overwrite
+          - task: CmdLine@2
+            displayName: Run full test suite
+            inputs:
+              script: |
+                pipenv run make test-integration
+          - task: CmdLine@2
+            inputs:
+              script: |
+                mkdir output-integration
+                mv unittest.xml output-integration/unittest.xml
+                mv .coverage output-integration/coverage
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: 'output-integration/'
+              artifact: 'coverage-integration'
+              publishLocation: 'pipeline'
       - job: coverage_e2e
         pool:
           name: coventry
@@ -222,13 +256,6 @@ stages:
               dockerComposeFile: 'scripts/ci.docker-compose.yml'
               action: 'Run services'
               buildImages: false
-          - task: CmdLine@2
-            displayName: Install K3d and prepare
-            inputs:
-              script: |
-                wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
-                k3d cluster create
-                k3d kubeconfig write -o ~/.kube/config --overwrite
           - task: CmdLine@2
             inputs:
               script: |
@@ -238,7 +265,7 @@ stages:
           - task: DockerCompose@0
             displayName: Run ChromeDriver
             inputs:
-              dockerComposeFile: 'e2e/ci.docker-compose.yml'
+              dockerComposeFile: 'tests/e2e/ci.docker-compose.yml'
               action: 'Run a specific service'
               serviceName: 'chrome'
           - task: CmdLine@2
@@ -252,8 +279,7 @@ stages:
             displayName: Run full test suite
             inputs:
               script: |
-                export PB_TEST_K8S=true
-                pipenv run coverage run ./manage.py test e2e -v 3 --failfast
+                pipenv run make test-e2e
           - task: CmdLine@2
             condition: always()
             displayName: Cleanup
@@ -291,6 +317,11 @@ stages:
               buildType: 'current'
               artifactName: 'coverage-e2e'
               path: "coverage-e2e/"
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              buildType: 'current'
+              artifactName: 'coverage-integration'
+              path: "coverage-integration/"
           - task: DownloadPipelineArtifact@2
             inputs:
               buildType: 'current'
@@ -305,7 +336,7 @@ stages:
                 sudo apt install -y libxmlsec1-dev pkg-config
                 sudo pip install -U wheel pipenv
                 pipenv install --dev
-                pipenv run coverage combine coverage-e2e/coverage coverage-unittest/coverage
+                pipenv run coverage combine coverage-e2e/coverage coverage-unittest/coverage coverage-integration/coverage
                 pipenv run coverage xml
                 pipenv run coverage html
           - task: PublishCodeCoverageResults@1
@@ -319,6 +350,7 @@ stages:
               testResultsFormat: 'JUnit'
               testResultsFiles: |
                 coverage-e2e/unittest.xml
+                coverage-integration/unittest.xml
                 coverage-unittest/unittest.xml
               mergeTestResults: true
           - task: CmdLine@2

passbook/outposts/apps.py

@@ -28,11 +28,12 @@ class PassbookOutpostConfig(AppConfig):
     def ready(self):
         import_module("passbook.outposts.signals")
         try:
-            self.init_local_connection()
+            PassbookOutpostConfig.init_local_connection()
         except ProgrammingError:
             pass
 
-    def init_local_connection(self):
+    @staticmethod
+    def init_local_connection():
         """Check if local kubernetes or docker connections should be created"""
         from passbook.outposts.models import (
             KubernetesServiceConnection,

passbook/outposts/tests.py

@@ -1,17 +1,10 @@
 """outpost tests"""
-from os import environ
-from unittest.case import skipUnless
-
 from django.test import TestCase
 from guardian.models import UserObjectPermission
 
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow
-from passbook.lib.config import CONFIG
-from passbook.outposts.controllers.k8s.base import NeedsUpdate
-from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler
-from passbook.outposts.controllers.kubernetes import KubernetesController
-from passbook.outposts.models import KubernetesServiceConnection, Outpost, OutpostType
+from passbook.outposts.models import Outpost, OutpostType
 from passbook.providers.proxy.models import ProxyProvider
 
 
@@ -64,51 +57,3 @@ class OutpostTests(TestCase):
         permissions = UserObjectPermission.objects.filter(user=outpost.user)
         self.assertEqual(len(permissions), 1)
         self.assertEqual(permissions[0].object_pk, str(outpost.pk))
-
-
-@skipUnless("PB_TEST_K8S" in environ, "Kubernetes test cluster required")
-class OutpostKubernetesTests(TestCase):
-    """Test Kubernetes Controllers"""
-
-    def setUp(self):
-        super().setUp()
-        self.provider: ProxyProvider = ProxyProvider.objects.create(
-            name="test",
-            internal_host="http://localhost",
-            external_host="http://localhost",
-            authorization_flow=Flow.objects.first(),
-        )
-        self.service_connection = KubernetesServiceConnection.objects.first()
-        self.outpost: Outpost = Outpost.objects.create(
-            name="test",
-            type=OutpostType.PROXY,
-            service_connection=self.service_connection,
-        )
-        self.outpost.providers.add(self.provider)
-        self.outpost.save()
-
-    def test_deployment_reconciler(self):
-        """test that deployment requires update"""
-        controller = KubernetesController(self.outpost, self.service_connection)
-        deployment_reconciler = DeploymentReconciler(controller)
-
-        self.assertIsNotNone(deployment_reconciler.retrieve())
-
-        config = self.outpost.config
-        config.kubernetes_replicas = 3
-        self.outpost.config = config
-
-        with self.assertRaises(NeedsUpdate):
-            deployment_reconciler.reconcile(
-                deployment_reconciler.retrieve(),
-                deployment_reconciler.get_reference_object(),
-            )
-
-        with CONFIG.patch("outposts.docker_image_base", "test"):
-            with self.assertRaises(NeedsUpdate):
-                deployment_reconciler.reconcile(
-                    deployment_reconciler.retrieve(),
-                    deployment_reconciler.get_reference_object(),
-                )
-
-        deployment_reconciler.delete(deployment_reconciler.get_reference_object())

tests/e2e/test_flows_enroll.py

@@ -8,13 +8,13 @@ from docker.types import Healthcheck
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as ec
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
 from passbook.stages.email.models import EmailStage, EmailTemplates
 from passbook.stages.identification.models import IdentificationStage
 from passbook.stages.prompt.models import FieldTypes, Prompt, PromptStage
 from passbook.stages.user_login.models import UserLoginStage
 from passbook.stages.user_write.models import UserWriteStage
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 
 @skipUnless(platform.startswith("linux"), "requires local docker")

tests/e2e/test_flows_login.py

@@ -5,7 +5,7 @@ from unittest.case import skipUnless
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 
-from e2e.utils import USER, SeleniumTestCase, retry
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 
 @skipUnless(platform.startswith("linux"), "requires local docker")

tests/e2e/test_flows_otp.py

@@ -12,9 +12,9 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.flows.models import Flow, FlowStageBinding
 from passbook.stages.otp_validate.models import OTPValidateStage
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 
 @skipUnless(platform.startswith("linux"), "requires local docker")

tests/e2e/test_flows_stage_setup.py

@@ -5,11 +5,11 @@ from unittest.case import skipUnless
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import User
 from passbook.flows.models import Flow, FlowDesignation
 from passbook.providers.oauth2.generators import generate_client_secret
 from passbook.stages.password.models import PasswordStage
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 
 @skipUnless(platform.startswith("linux"), "requires local docker")

tests/e2e/test_provider_oauth2_github.py

@@ -9,7 +9,6 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import Application
 from passbook.flows.models import Flow
 from passbook.policies.expression.models import ExpressionPolicy
@@ -19,6 +18,7 @@ from passbook.providers.oauth2.generators import (
     generate_client_secret,
 )
 from passbook.providers.oauth2.models import ClientTypes, OAuth2Provider, ResponseTypes
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 
 @skipUnless(platform.startswith("linux"), "requires local docker")

tests/e2e/test_provider_oauth2_grafana.py

@@ -10,7 +10,6 @@ from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import Application
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow
@@ -31,6 +30,7 @@ from passbook.providers.oauth2.models import (
     ResponseTypes,
     ScopeMapping,
 )
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 LOGGER = get_logger()
 APPLICATION_SLUG = "grafana"

tests/e2e/test_provider_oauth2_oidc.py

@@ -12,7 +12,6 @@ from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import Application
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow
@@ -33,6 +32,7 @@ from passbook.providers.oauth2.models import (
     ResponseTypes,
     ScopeMapping,
 )
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 LOGGER = get_logger()
 

tests/e2e/test_provider_proxy.py

@@ -11,10 +11,10 @@ from docker.models.containers import Container
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook import __version__
 from passbook.core.models import Application
 from passbook.flows.models import Flow
+from passbook.outposts.apps import PassbookOutpostConfig
 from passbook.outposts.models import (
     DockerServiceConnection,
     Outpost,
@@ -22,6 +22,7 @@ from passbook.outposts.models import (
     OutpostType,
 )
 from passbook.providers.proxy.models import ProxyProvider
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 
 @skipUnless(platform.startswith("linux"), "requires local docker")
@@ -113,6 +114,7 @@ class TestProviderProxyConnect(ChannelsLiveServerTestCase):
     @retry()
     def test_proxy_connectivity(self):
         """Test proxy connectivity over websocket"""
+        PassbookOutpostConfig.init_local_connection()
         SeleniumTestCase().apply_default_data()
         proxy: ProxyProvider = ProxyProvider.objects.create(
             name="proxy_provider",

tests/e2e/test_provider_saml.py

@@ -12,7 +12,6 @@ from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 
-from e2e.utils import USER, SeleniumTestCase, retry
 from passbook.core.models import Application
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow
@@ -23,6 +22,7 @@ from passbook.providers.saml.models import (
     SAMLPropertyMapping,
     SAMLProvider,
 )
+from tests.e2e.utils import USER, SeleniumTestCase, retry
 
 LOGGER = get_logger()
 

tests/e2e/test_source_oauth.py

@@ -14,13 +14,13 @@ from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 from yaml import safe_dump
 
-from e2e.utils import SeleniumTestCase, retry
 from passbook.flows.models import Flow
 from passbook.providers.oauth2.generators import (
     generate_client_id,
     generate_client_secret,
 )
 from passbook.sources.oauth.models import OAuthSource
+from tests.e2e.utils import SeleniumTestCase, retry
 
 CONFIG_PATH = "/tmp/dex.yml"
 LOGGER = get_logger()

tests/e2e/test_source_saml.py

@@ -10,10 +10,10 @@ from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from structlog import get_logger
 
-from e2e.utils import SeleniumTestCase, retry
 from passbook.crypto.models import CertificateKeyPair
 from passbook.flows.models import Flow
 from passbook.sources.saml.models import SAMLBindingTypes, SAMLSource
+from tests.e2e.utils import SeleniumTestCase, retry
 
 LOGGER = get_logger()
 

tests/integration/test_outposts_kubernetes.py

@@ -0,0 +1,60 @@
+"""outpost tests"""
+from django.test import TestCase
+
+from passbook.flows.models import Flow
+from passbook.lib.config import CONFIG
+from passbook.outposts.apps import PassbookOutpostConfig
+from passbook.outposts.controllers.k8s.base import NeedsUpdate
+from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler
+from passbook.outposts.controllers.kubernetes import KubernetesController
+from passbook.outposts.models import KubernetesServiceConnection, Outpost, OutpostType
+from passbook.providers.proxy.models import ProxyProvider
+
+
+class OutpostKubernetesTests(TestCase):
+    """Test Kubernetes Controllers"""
+
+    def setUp(self):
+        super().setUp()
+        # Ensure that local connection have been created
+        PassbookOutpostConfig.init_local_connection()
+        self.provider: ProxyProvider = ProxyProvider.objects.create(
+            name="test",
+            internal_host="http://localhost",
+            external_host="http://localhost",
+            authorization_flow=Flow.objects.first(),
+        )
+        self.service_connection = KubernetesServiceConnection.objects.first()
+        self.outpost: Outpost = Outpost.objects.create(
+            name="test",
+            type=OutpostType.PROXY,
+            service_connection=self.service_connection,
+        )
+        self.outpost.providers.add(self.provider)
+        self.outpost.save()
+
+    def test_deployment_reconciler(self):
+        """test that deployment requires update"""
+        controller = KubernetesController(self.outpost, self.service_connection)
+        deployment_reconciler = DeploymentReconciler(controller)
+
+        self.assertIsNotNone(deployment_reconciler.retrieve())
+
+        config = self.outpost.config
+        config.kubernetes_replicas = 3
+        self.outpost.config = config
+
+        with self.assertRaises(NeedsUpdate):
+            deployment_reconciler.reconcile(
+                deployment_reconciler.retrieve(),
+                deployment_reconciler.get_reference_object(),
+            )
+
+        with CONFIG.patch("outposts.docker_image_base", "test"):
+            with self.assertRaises(NeedsUpdate):
+                deployment_reconciler.reconcile(
+                    deployment_reconciler.retrieve(),
+                    deployment_reconciler.get_reference_object(),
+                )
+
+        deployment_reconciler.delete(deployment_reconciler.get_reference_object())

tests/integration/test_proxy_kubernetes.py

@@ -1,20 +1,21 @@
 """Test Controllers"""
-from os import environ
-from unittest import skipUnless
-
 import yaml
 from django.test import TestCase
 
 from passbook.flows.models import Flow
+from passbook.outposts.apps import PassbookOutpostConfig
 from passbook.outposts.models import KubernetesServiceConnection, Outpost, OutpostType
 from passbook.providers.proxy.controllers.kubernetes import ProxyKubernetesController
 from passbook.providers.proxy.models import ProxyProvider
 
 
-@skipUnless("PB_TEST_K8S" in environ, "Kubernetes test cluster required")
 class TestControllers(TestCase):
     """Test Controllers"""
 
+    def setUp(self):
+        # Ensure that local connection have been created
+        PassbookOutpostConfig.init_local_connection()
+
     def test_kubernetes_controller_static(self):
         """Test Kubernetes Controller"""
         provider: ProxyProvider = ProxyProvider.objects.create(

tests/setup.sh

@@ -1,20 +1,20 @@
 #!/bin/bash -x
 # Setup docker & compose
-curl -fsSL https://get.docker.com | bash
+# curl -fsSL https://get.docker.com | bash
 sudo usermod -a -G docker ubuntu
 sudo curl -L "https://github.com/docker/compose/releases/download/1.26.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
 # Setup nodejs
-curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -
+curl -sL https://deb.nodesource.com/setup_15.x | sudo -E bash -
 sudo apt-get install -y nodejs
-sudo npm install -g yarn
+# Setup k3d
+curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
 # Setup python
 sudo apt install -y python3.9 python3-pip libxmlsec1-dev pkg-config
 # Setup docker
 sudo pip3 install pipenv
 
-cd e2e
+cd tests/e2e
 sudo docker-compose up -d
-cd ..
+cd ../..
 pipenv sync --dev
-pipenv shell