e2e: add apply_default_data to load data from migrations after tables have been truncated

.github/workflows/ci.yml

@@ -130,6 +130,23 @@ jobs:
       - uses: codecov/codecov-action@v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+  e2e:
+    needs:
+      - pylint
+      - black
+      - prospector
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Setup test containers
+        run: |
+          cd e2e
+          docker-compose pull
+          docker-compose up -d
+          docker-compose exec passbook pip install -r /app/requirements-dev.txt
+      - name: Run e2e tests
+        run: |
+          docker-compose exec passbook ./manage.py test e2e
   # Build
   build-server:
     needs:

.github/workflows/release.yml

@@ -82,7 +82,6 @@ jobs:
       - uses: actions/checkout@v1
       - name: Run test suite in final docker images
         run: |
-          export PASSBOOK_DOMAIN=localhost
           docker-compose pull
           docker-compose up --no-start
           docker-compose start postgresql redis

.github/workflows/tag.yml

@@ -13,7 +13,6 @@ jobs:
       - uses: actions/checkout@master
       - name: Pre-release test
         run: |
-          export PASSBOOK_DOMAIN=localhost
           docker-compose pull
           docker build \
             --no-cache \

docker.env.yml

@@ -0,0 +1,9 @@
+debug: true
+postgresql:
+  user: postgres
+  host: postgresql
+
+redis:
+  host: redis
+
+log_level: debug

docs/installation/docker-compose.md

@@ -11,12 +11,6 @@ This installation Method is for test-setups and small-scale productive setups.
 
 Download the latest `docker-compose.yml` from [here](https://raw.githubusercontent.com/BeryJu/passbook/master/docker-compose.yml). Place it in a directory of your choice.
 
-passbook needs to know it's primary URL to create links in E-Mails and set cookies, so you have to run the following command:
-
-```
-export PASSBOOK_DOMAIN=domain.tld # this can be any domain or IP, it just needs to point to passbook.
-```
-
 The compose file references the current latest version, which can be overridden with the `SERVER_TAG` Environment variable.
 
 If you plan to use this setup for production, it is also advised to change the PostgreSQL Password by setting `PG_PASS` to a password of your choice.

e2e/docker-compose.yml

@@ -12,3 +12,21 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - /tmp/videos:/home/seluser/videos
     privileged: true
+  postgresql:
+    image: postgres:11
+    restart: always
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_DB: passbook
+  redis:
+    image: redis
+    restart: always
+  passbook:
+    image: beryju/passbook
+    command: /bin/bash -c "sleep infinity"
+    volumes:
+      - ../:/testing
+    environment:
+      PASSBOOK_ENV: docker
+    user: root
+    working_dir: /testing

e2e/test_enroll_2_step.py

@@ -10,20 +10,21 @@ from passbook.policies.models import PolicyBinding
 from passbook.stages.prompt.models import FieldTypes, Prompt, PromptStage
 from passbook.stages.user_login.models import UserLoginStage
 from passbook.stages.user_write.models import UserWriteStage
-
+from passbook.stages.identification.models import IdentificationStage
+from e2e.utils import apply_default_data
 
 class TestEnroll2Step(StaticLiveServerTestCase):
     """Test 2-step enroll flow"""
 
-    host = "0.0.0.0"
+    host = "passbook"
-    port = 8001
 
     def setUp(self):
         self.driver = webdriver.Remote(
-            command_executor="http://localhost:4444/wd/hub",
+            command_executor="http://hub:4444/wd/hub",
             desired_capabilities=DesiredCapabilities.CHROME,
         )
-        self.driver.implicitly_wait(2)
+        self.driver.implicitly_wait(5)
+        apply_default_data()
 
     def tearDown(self):
         super().tearDown()
@@ -66,7 +67,7 @@ class TestEnroll2Step(StaticLiveServerTestCase):
         # Password checking policy
         password_policy = ExpressionPolicy.objects.create(
             name="policy-enrollment-password-equals",
-            expression="{{ request.context.password == request.context.password_repeat }}",
+            expression="return request.context['password'] == request.context['password_repeat']",
         )
         PolicyBinding.objects.create(
             target=first_stage, policy=password_policy, order=0
@@ -78,11 +79,16 @@ class TestEnroll2Step(StaticLiveServerTestCase):
             designation=FlowDesignation.ENROLLMENT,
         )
 
+        # Attach enrollment flow to identification stage
+        ident_stage: IdentificationStage = IdentificationStage.objects.first()
+        ident_stage.enrollment_flow = flow
+        ident_stage.save()
+
         FlowStageBinding.objects.create(flow=flow, stage=first_stage, order=0)
         FlowStageBinding.objects.create(flow=flow, stage=second_stage, order=1)
         FlowStageBinding.objects.create(flow=flow, stage=user_write, order=2)
         FlowStageBinding.objects.create(flow=flow, stage=user_login, order=3)
-        self.driver.get(f"http://host.docker.internal:{self.port}")
+        self.driver.get(self.live_server_url)
         self.driver.find_element(By.CSS_SELECTOR, "[role=enroll]").click()
         self.driver.find_element(By.ID, "id_username").send_keys("foo")
         self.driver.find_element(By.ID, "id_password").send_keys("pbadmin")

e2e/test_login_default.py

@@ -1,38 +1,24 @@
 """test default login flow"""
-import string
-from random import SystemRandom
-
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
-from django.core.management import call_command
 from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.common.keys import Keys
-
+from e2e.utils import apply_default_data
-from passbook.core.models import User
 
 
 class TestLogin(StaticLiveServerTestCase):
     """test default login flow"""
 
-    host = "0.0.0.0"
+    host = "passbook"
-    port = 8000
 
     def setUp(self):
         self.driver = webdriver.Remote(
-            command_executor="http://localhost:4444/wd/hub",
+            command_executor="http://hub:4444/wd/hub",
             desired_capabilities=DesiredCapabilities.CHROME,
         )
-        self.driver.implicitly_wait(2)
+        self.driver.implicitly_wait(5)
-        self.password = "".join(
+        apply_default_data()
-            SystemRandom().choice(string.ascii_uppercase + string.digits)
-            for _ in range(8)
-        )
-        User.objects.create_superuser(
-            username="pbadmin", email="admin@example.tld", password=self.password
-        )
-        call_command("migrate", "--fake", "passbook_flows", "0001_initial")
-        call_command("migrate", "passbook_flows", "0002_default_flows")
 
     def tearDown(self):
         super().tearDown()
@@ -41,12 +27,12 @@ class TestLogin(StaticLiveServerTestCase):
     def test_login(self):
         """test default login flow"""
         self.driver.get(
-            f"http://host.docker.internal:{self.port}/flows/default-authentication-flow/?next=%2F"
+            f"{self.live_server_url}/flows/default-authentication-flow/"
         )
         self.driver.find_element(By.ID, "id_uid_field").click()
-        self.driver.find_element(By.ID, "id_uid_field").send_keys("admin@example.tld")
+        self.driver.find_element(By.ID, "id_uid_field").send_keys("pbadmin")
         self.driver.find_element(By.ID, "id_uid_field").send_keys(Keys.ENTER)
-        self.driver.find_element(By.ID, "id_password").send_keys(self.password)
+        self.driver.find_element(By.ID, "id_password").send_keys("pbadmin")
         self.driver.find_element(By.ID, "id_password").send_keys(Keys.ENTER)
         self.assertEqual(
             self.driver.find_element(By.XPATH, "//a[contains(@href, '/-/user/')]").text,

e2e/utils.py

@@ -0,0 +1,35 @@
+"""passbook e2e testing utilities"""
+
+from glob import glob
+from inspect import getmembers, isfunction
+from importlib.util import spec_from_file_location, module_from_spec
+from django.apps import apps
+from django.db import connection, transaction
+from django.db.utils import IntegrityError
+
+def apply_default_data():
+    """apply objects created by migrations after tables have been truncated"""
+    # Find all migration files
+    # load all functions
+    migration_files = glob("**/migrations/*.py", recursive=True)
+    matches = []
+    for migration in migration_files:
+        with open(migration, 'r+') as migration_file:
+            # Check if they have a `RunPython`
+            if "RunPython" in migration_file.read():
+                matches.append(migration)
+
+    with connection.schema_editor() as schema_editor:
+        for match in matches:
+            # Load module from file path
+            spec = spec_from_file_location("", match)
+            migration_module = module_from_spec(spec)
+            # pyright: reportGeneralTypeIssues=false
+            spec.loader.exec_module(migration_module)
+            # Call all functions from module
+            for _, func in getmembers(migration_module, isfunction):
+                with transaction.atomic():
+                    try:
+                        func(apps, schema_editor)
+                    except IntegrityError:
+                        pass

passbook/flows/migrations/0004_source_flows.py

@@ -7,15 +7,10 @@ from django.db.backends.base.schema import BaseDatabaseSchemaEditor
 from passbook.flows.models import FlowDesignation
 from passbook.stages.prompt.models import FieldTypes
 
-FLOW_POLICY_EXPRESSION = """{{ pb_is_sso_flow }}"""
+FLOW_POLICY_EXPRESSION = """return pb_is_sso_flow"""
-
+PROMPT_POLICY_EXPRESSION = (
-PROMPT_POLICY_EXPRESSION = """
+    """return 'username' in pb_flow_plan.context['prompt_data']"""
-{% if pb_flow_plan.context.prompt_data.username %}
+)
-False
-{% else %}
-True
-{% endif %}
-"""
 
 
 def create_default_source_enrollment_flow(

scripts/coverage.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -xe
-coverage run --concurrency=multiprocessing manage.py test --failfast
+coverage run --concurrency=multiprocessing manage.py test passbook --failfast
 coverage combine
 coverage html
 coverage report