deploy: rewrite docker files to be more stackable

.bumpversion.cfg

@@ -10,7 +10,7 @@ tag_name = version/{new_version}
 [bumpversion:part:release]
 optional_value = stable
 first_value = beta
-values = 
+values =
 	alpha
 	beta
 	stable
@@ -27,33 +27,4 @@ values =
 
 [bumpversion:file:passbook/__init__.py]
 
-[bumpversion:file:passbook/api/__init__.py]
+[bumpversion:file:passbook/core/nginx.conf]
-
-[bumpversion:file:passbook/core/__init__.py]
-
-[bumpversion:file:passbook/admin/__init__.py]
-
-[bumpversion:file:passbook/captcha_factor/__init__.py]
-
-[bumpversion:file:passbook/oauth_client/__init__.py]
-
-[bumpversion:file:passbook/ldap/__init__.py]
-
-[bumpversion:file:passbook/lib/__init__.py]
-
-[bumpversion:file:passbook/hibp_policy/__init__.py]
-
-[bumpversion:file:passbook/password_expiry_policy/__init__.py]
-
-[bumpversion:file:passbook/saml_idp/__init__.py]
-
-[bumpversion:file:passbook/audit/__init__.py]
-
-[bumpversion:file:passbook/oauth_provider/__init__.py]
-
-[bumpversion:file:passbook/otp/__init__.py]
-
-[bumpversion:file:passbook/app_gw/__init__.py]
-
-[bumpversion:file:passbook/suspicious_policy/__init__.py]
-

.gitlab-ci.yml

@@ -1,14 +1,11 @@
 # Global Variables
 stages:
-  - build-buildimage
+  - build-base-image
+  - build-dev-image
   - test
   - build
-  - docs
+  - package
-  - deploy
+image: docker.beryju.org/passbook/dev:latest
-image: docker.beryju.org/passbook/build-base:latest
-services:
-  - postgres:latest
-  - redis:latest
 
 variables:
   POSTGRES_DB: passbook
@@ -16,51 +13,78 @@ variables:
   POSTGRES_PASSWORD: "EK-5jnKfjrGRm<77"
 
 before_script:
-  # Ensure all dependencies are installed, even those not included in passbook/build-base
+  # Ensure all dependencies are installed, even those not included in passbook/dev
+  - pip install -r requirements.txt
   - pip install -r requirements-dev.txt
 
-create-build-image:
+create-base-image:
   image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [""]
   before_script:
     - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json
   script:
-    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.build-base --destination docker.beryju.org/passbook/build-base:latest --destination docker.beryju.org/passbook/build-base:0.2.6-beta
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.base --destination docker.beryju.org/passbook/base:latest --destination docker.beryju.org/passbook/base:0.2.6-beta
-  stage: build-buildimage
+  stage: build-base-image
   only:
     refs:
       - tags
       - /^version/.*$/
 
+build-dev-image:
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  before_script:
+    - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json
+  script:
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.dev --destination docker.beryju.org/passbook/dev:latest --destination docker.beryju.org/passbook/dev:0.2.6-beta
+  stage: build-dev-image
+  only:
+    refs:
+      - tags
+      - /^version/.*$/
+
+
 isort:
   script:
     - isort -c -sg env
   stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 migrations:
   script:
     - python manage.py migrate
   stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 prospector:
   script:
     - prospector
   stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 pylint:
   script:
-    - pylint passbook
+    - pylint p2
   stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 coverage:
   script:
-    - python manage.py collectstatic --no-input
     - coverage run manage.py test
     - coverage report
+    - coverage html
   stage: test
-bandit:
+  services:
-  script:
+  - postgres:latest
-    - bandit -r passbook
+  - redis:latest
-  stage: test
 
-package-docker:
+package-passbook-server:
   image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [""]
@@ -72,42 +96,37 @@ package-docker:
   only:
     - tags
     - /^version/.*$/
-package-helm:
+build-passbook-static:
   stage: build
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  before_script:
+    - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json
   script:
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.static --destination docker.beryju.org/passbook/static:latest --destination docker.beryju.org/passbook/static:0.2.6-beta
+  only:
+    - tags
+    - /^version/.*$/
+  # running collectstatic fully initialises django, hence we need that databases
+  services:
+    - postgres:latest
+    - redis:latest
+
+package-helm:
+  image: debian:stretch-slim
+  stage: package
+  before_script:
+    - apt update && apt install -y curl
     - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
+  script:
     - helm init --client-only
+    - helm dependency build helm/passbook
     - helm package helm/passbook
   artifacts:
     paths:
       - passbook-*.tgz
-    expire_in: 2 days
+    expire_in: 1 week
   only:
     - tags
     - /^version/.*$/
-
-package-client-package-allauth:
-  script:
-    - cd client-packages/allauth
-    - python setup.py sdist
-    - twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD dist/*
-  stage: build
-  only:
-    refs:
-      - tags
-      - /^version/.*$/
-    changes:
-      - client-packages/allauth/**
-
-package-client-package-sentry:
-  script:
-    - cd client-packages/sentry-auth-passbook
-    - python setup.py sdist
-    - twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD dist/*
-  stage: build
-  only:
-    refs:
-      - tags
-      - /^version/.*$/
-    changes:
-      - client-packages/sentry-auth-passbook/**

Dockerfile

@@ -1,34 +1,8 @@
-FROM python:3.6-slim-stretch as build
+FROM docker.beryju.org/passbook/base:latest
 
 COPY ./passbook/ /app/passbook
 COPY ./manage.py /app/
-COPY ./requirements.txt /app/
-
-WORKDIR /app/
-
-RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \
-    mkdir /app/static/ && \
-    pip install -r requirements.txt && \
-    pip install psycopg2 && \
-    ./manage.py collectstatic --no-input && \
-    apt-get remove --purge -y build-essential && \
-    apt-get autoremove --purge -y
-
-FROM python:3.6-slim-stretch
-
-COPY ./passbook/ /app/passbook
-COPY ./manage.py /app/
-COPY ./requirements.txt /app/
-COPY --from=build /app/static /app/static/
-
-WORKDIR /app/
-
-RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \
-    pip install -r requirements.txt && \
-    pip install psycopg2 && \
-    adduser --system --home /app/ passbook && \
-    chown -R passbook /app/ && \
-    apt-get remove --purge -y build-essential && \
-    apt-get autoremove --purge -y
 
 USER passbook
+
+WORKDIR /app/

Dockerfile.base

@@ -0,0 +1,11 @@
+FROM python:3.7-alpine
+
+COPY ./requirements.txt /app/
+
+WORKDIR /app/
+
+RUN apk update && \
+    apk add --no-cache openssl-dev build-base libxml2-dev libxslt-dev libffi-dev gcc musl-dev libgcc zlib-dev postgresql-dev && \
+    pip install -r /app/requirements.txt  --no-cache-dir && \
+    adduser -S passbook && \
+    chown -R passbook /app

Dockerfile.build-base

@@ -1,12 +0,0 @@
-FROM python:3.6
-
-COPY ./passbook/ /app/passbook
-COPY ./client-packages/ /app/client-packages
-COPY ./requirements.txt /app/
-COPY ./requirements-dev.txt /app/
-
-WORKDIR /app/
-
-RUN apt-get update && apt-get install libssl-dev libffi-dev libpq-dev -y && \
-     pip install -U -r requirements-dev.txt && \
-     rm -rf /app/*

Dockerfile.dev

@@ -0,0 +1,5 @@
+FROM docker.beryju.org/passbook/base:latest
+
+COPY ./requirements-dev.txt /app/
+
+RUN pip install -r /app/requirements-dev.txt  --no-cache-dir

Dockerfile.static

@@ -0,0 +1,14 @@
+FROM docker.beryju.org/passbook/dev:latest as static-build
+
+COPY ./passbook/ /app/passbook
+COPY ./manage.py /app/
+COPY ./requirements.txt /app/
+
+WORKDIR /app/
+
+RUN ./manage.py collectstatic --no-input
+
+FROM nginx:latest
+
+COPY --from=static-build /app/static /static/_/static/
+COPY ./passbook/core/nginx.conf /etc/nginx/nginx.conf

requirements-dev.txt

@@ -1,5 +1,4 @@
 -r requirements.txt
--r client-packages/allauth/requirements.txt
 coverage
 isort
 astroid==2.0.4
@@ -14,3 +13,4 @@ autopep8
 bandit
 bumpversion
 twine
+grpcio-tools

requirements.txt

@@ -1,11 +1,51 @@
--r passbook/root/requirements.txt
+# Root requirements
--r passbook/oauth_client/requirements.txt
+celery
--r passbook/ldap/requirements.txt
+colorlog
--r passbook/saml_idp/requirements.txt
+cherrypy
--r passbook/otp/requirements.txt
+django-ipware
--r passbook/oauth_provider/requirements.txt
+django-model-utils
--r passbook/captcha_factor/requirements.txt
+django-redis
--r passbook/admin/requirements.txt
+django>=2.0
--r passbook/api/requirements.txt
+idna<2.8,>=2.5
--r passbook/app_gw/requirements.txt
+markdown
--r passbook/oidc_provider/requirements.txt
+psycopg2
+PyYAML
+sentry-sdk
+pip
+urllib3<1.25,>=1.21.1
+grpcio
+grpcio-reflection
+protobuf
+# OAuth Client
+requests_oauthlib>=0.4.2
+oauthlib>=2.0.6
+# LDAP Client
+ldap3
+# SAML IDP
+beautifulsoup4>=4.6.0
+lxml>=3.8.0
+signxml
+defusedxml
+PyCryptodome
+# OTP
+django_otp
+qrcode
+# OAuth Provider
+django-oauth-toolkit
+django-cors-middleware
+# ReCaptcha
+django-recaptcha
+# API
+drf_yasg
+djangorestframework==3.9.4
+django-filters
+# AppGW
+django-revproxy
+urllib3[secure]
+channels
+service_identity
+websocket-client
+daphne<2.3.0
+asgiref~=2.3
+# OIDC Provider
+django-oidc-provider