deploy: rewrite docker files to be more stackable

2019-07-22 15:18:33 +02:00 · 2019-07-22 15:18:33 +02:00 · 17575ed921
parent b1b1a27444
commit 17575ed921
9 changed files with 152 additions and 130 deletions
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@ -10,7 +10,7 @@ tag_name = version/{new_version}
 [bumpversion:part:release]
 optional_value = stable
 first_value = beta
-values = 
+values =
 	alpha
 	beta
 	stable
@ -27,33 +27,4 @@ values =

 [bumpversion:file:passbook/__init__.py]

-[bumpversion:file:passbook/api/__init__.py]
-
-[bumpversion:file:passbook/core/__init__.py]
-
-[bumpversion:file:passbook/admin/__init__.py]
-
-[bumpversion:file:passbook/captcha_factor/__init__.py]
-
-[bumpversion:file:passbook/oauth_client/__init__.py]
-
-[bumpversion:file:passbook/ldap/__init__.py]
-
-[bumpversion:file:passbook/lib/__init__.py]
-
-[bumpversion:file:passbook/hibp_policy/__init__.py]
-
-[bumpversion:file:passbook/password_expiry_policy/__init__.py]
-
-[bumpversion:file:passbook/saml_idp/__init__.py]
-
-[bumpversion:file:passbook/audit/__init__.py]
-
-[bumpversion:file:passbook/oauth_provider/__init__.py]
-
-[bumpversion:file:passbook/otp/__init__.py]
-
-[bumpversion:file:passbook/app_gw/__init__.py]
-
-[bumpversion:file:passbook/suspicious_policy/__init__.py]
-
+[bumpversion:file:passbook/core/nginx.conf]
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,14 +1,11 @@
 # Global Variables
 stages:
-  - build-buildimage
+  - build-base-image
+  - build-dev-image
  - test
  - build
-  - docs
-  - deploy
-image: docker.beryju.org/passbook/build-base:latest
-services:
-  - postgres:latest
-  - redis:latest
+  - package
+image: docker.beryju.org/passbook/dev:latest

 variables:
  POSTGRES_DB: passbook
@ -16,51 +13,78 @@ variables:
  POSTGRES_PASSWORD: "EK-5jnKfjrGRm<77"

 before_script:
-  # Ensure all dependencies are installed, even those not included in passbook/build-base
+  # Ensure all dependencies are installed, even those not included in passbook/dev
+  - pip install -r requirements.txt
  - pip install -r requirements-dev.txt

-create-build-image:
+create-base-image:
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  before_script:
    - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json
  script:
-    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.build-base --destination docker.beryju.org/passbook/build-base:latest --destination docker.beryju.org/passbook/build-base:0.2.6-beta
-  stage: build-buildimage
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.base --destination docker.beryju.org/passbook/base:latest --destination docker.beryju.org/passbook/base:0.2.6-beta
+  stage: build-base-image
  only:
    refs:
      - tags
      - /^version/.*$/

+build-dev-image:
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  before_script:
+    - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json
+  script:
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.dev --destination docker.beryju.org/passbook/dev:latest --destination docker.beryju.org/passbook/dev:0.2.6-beta
+  stage: build-dev-image
+  only:
+    refs:
+      - tags
+      - /^version/.*$/
+
+
 isort:
  script:
    - isort -c -sg env
  stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 migrations:
  script:
    - python manage.py migrate
  stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 prospector:
  script:
    - prospector
  stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 pylint:
  script:
-    - pylint passbook
+    - pylint p2
  stage: test
+  services:
+  - postgres:latest
+  - redis:latest
 coverage:
  script:
-    - python manage.py collectstatic --no-input
    - coverage run manage.py test
    - coverage report
+    - coverage html
  stage: test
-bandit:
-  script:
-    - bandit -r passbook
-  stage: test
+  services:
+  - postgres:latest
+  - redis:latest

-package-docker:
+package-passbook-server:
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
@ -72,42 +96,37 @@ package-docker:
  only:
    - tags
    - /^version/.*$/
-package-helm:
+build-passbook-static:
  stage: build
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  before_script:
+    - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json
  script:
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.static --destination docker.beryju.org/passbook/static:latest --destination docker.beryju.org/passbook/static:0.2.6-beta
+  only:
+    - tags
+    - /^version/.*$/
+  # running collectstatic fully initialises django, hence we need that databases
+  services:
+    - postgres:latest
+    - redis:latest
+
+package-helm:
+  image: debian:stretch-slim
+  stage: package
+  before_script:
+    - apt update && apt install -y curl
    - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
+  script:
    - helm init --client-only
+    - helm dependency build helm/passbook
    - helm package helm/passbook
  artifacts:
    paths:
      - passbook-*.tgz
-    expire_in: 2 days
+    expire_in: 1 week
  only:
    - tags
    - /^version/.*$/
-
-package-client-package-allauth:
-  script:
-    - cd client-packages/allauth
-    - python setup.py sdist
-    - twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD dist/*
-  stage: build
-  only:
-    refs:
-      - tags
-      - /^version/.*$/
-    changes:
-      - client-packages/allauth/**
-
-package-client-package-sentry:
-  script:
-    - cd client-packages/sentry-auth-passbook
-    - python setup.py sdist
-    - twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD dist/*
-  stage: build
-  only:
-    refs:
-      - tags
-      - /^version/.*$/
-    changes:
-      - client-packages/sentry-auth-passbook/**
--- a/32
+++ b/32
@ -1,34 +1,8 @@
-FROM python:3.6-slim-stretch as build
+FROM docker.beryju.org/passbook/base:latest

 COPY ./passbook/ /app/passbook
 COPY ./manage.py /app/
-COPY ./requirements.txt /app/
-
-WORKDIR /app/
-
-RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \
-    mkdir /app/static/ && \
-    pip install -r requirements.txt && \
-    pip install psycopg2 && \
-    ./manage.py collectstatic --no-input && \
-    apt-get remove --purge -y build-essential && \
-    apt-get autoremove --purge -y
-
-FROM python:3.6-slim-stretch
-
-COPY ./passbook/ /app/passbook
-COPY ./manage.py /app/
-COPY ./requirements.txt /app/
-COPY --from=build /app/static /app/static/
-
-WORKDIR /app/
-
-RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \
-    pip install -r requirements.txt && \
-    pip install psycopg2 && \
-    adduser --system --home /app/ passbook && \
-    chown -R passbook /app/ && \
-    apt-get remove --purge -y build-essential && \
-    apt-get autoremove --purge -y

 USER passbook
+
+WORKDIR /app/
--- a/Dockerfile.base
+++ b/Dockerfile.base
@ -0,0 +1,11 @@
+FROM python:3.7-alpine
+
+COPY ./requirements.txt /app/
+
+WORKDIR /app/
+
+RUN apk update && \
+    apk add --no-cache openssl-dev build-base libxml2-dev libxslt-dev libffi-dev gcc musl-dev libgcc zlib-dev postgresql-dev && \
+    pip install -r /app/requirements.txt  --no-cache-dir && \
+    adduser -S passbook && \
+    chown -R passbook /app
--- a/Dockerfile.build-base
+++ b/Dockerfile.build-base
@ -1,12 +0,0 @@
-FROM python:3.6
-
-COPY ./passbook/ /app/passbook
-COPY ./client-packages/ /app/client-packages
-COPY ./requirements.txt /app/
-COPY ./requirements-dev.txt /app/
-
-WORKDIR /app/
-
-RUN apt-get update && apt-get install libssl-dev libffi-dev libpq-dev -y && \
-     pip install -U -r requirements-dev.txt && \
-     rm -rf /app/*
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@ -0,0 +1,5 @@
+FROM docker.beryju.org/passbook/base:latest
+
+COPY ./requirements-dev.txt /app/
+
+RUN pip install -r /app/requirements-dev.txt  --no-cache-dir
--- a/Dockerfile.static
+++ b/Dockerfile.static
@ -0,0 +1,14 @@
+FROM docker.beryju.org/passbook/dev:latest as static-build
+
+COPY ./passbook/ /app/passbook
+COPY ./manage.py /app/
+COPY ./requirements.txt /app/
+
+WORKDIR /app/
+
+RUN ./manage.py collectstatic --no-input
+
+FROM nginx:latest
+
+COPY --from=static-build /app/static /static/_/static/
+COPY ./passbook/core/nginx.conf /etc/nginx/nginx.conf
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@ -1,5 +1,4 @@
 -r requirements.txt
-r client-packages/allauth/requirements.txt
 coverage
 isort
 astroid==2.0.4
@ -14,3 +13,4 @@ autopep8
 bandit
 bumpversion
 twine
+grpcio-tools
--- a/requirements.txt
+++ b/requirements.txt
@ -1,11 +1,51 @@
-r passbook/root/requirements.txt
-r passbook/oauth_client/requirements.txt
-r passbook/ldap/requirements.txt
-r passbook/saml_idp/requirements.txt
-r passbook/otp/requirements.txt
-r passbook/oauth_provider/requirements.txt
-r passbook/captcha_factor/requirements.txt
-r passbook/admin/requirements.txt
-r passbook/api/requirements.txt
-r passbook/app_gw/requirements.txt
-r passbook/oidc_provider/requirements.txt
+# Root requirements
+celery
+colorlog
+cherrypy
+django-ipware
+django-model-utils
+django-redis
+django>=2.0
+idna<2.8,>=2.5
+markdown
+psycopg2
+PyYAML
+sentry-sdk
+pip
+urllib3<1.25,>=1.21.1
+grpcio
+grpcio-reflection
+protobuf
+# OAuth Client
+requests_oauthlib>=0.4.2
+oauthlib>=2.0.6
+# LDAP Client
+ldap3
+# SAML IDP
+beautifulsoup4>=4.6.0
+lxml>=3.8.0
+signxml
+defusedxml
+PyCryptodome
+# OTP
+django_otp
+qrcode
+# OAuth Provider
+django-oauth-toolkit
+django-cors-middleware
+# ReCaptcha
+django-recaptcha
+# API
+drf_yasg
+djangorestframework==3.9.4
+django-filters
+# AppGW
+django-revproxy
+urllib3[secure]
+channels
+service_identity
+websocket-client
+daphne<2.3.0
+asgiref~=2.3
+# OIDC Provider
+django-oidc-provider