From 17575ed92195fdc5ea77cdd22f64802e96e7c6fe Mon Sep 17 00:00:00 2001 From: "Langhammer, Jens" Date: Mon, 22 Jul 2019 15:18:33 +0200 Subject: [PATCH] deploy: rewrite docker files to be more stackable --- .bumpversion.cfg | 33 +------------ .gitlab-ci.yml | 111 +++++++++++++++++++++++++----------------- Dockerfile | 32 ++---------- Dockerfile.base | 11 +++++ Dockerfile.build-base | 12 ----- Dockerfile.dev | 5 ++ Dockerfile.static | 14 ++++++ requirements-dev.txt | 2 +- requirements.txt | 62 ++++++++++++++++++----- 9 files changed, 152 insertions(+), 130 deletions(-) create mode 100644 Dockerfile.base delete mode 100644 Dockerfile.build-base create mode 100644 Dockerfile.dev create mode 100644 Dockerfile.static diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 353bef930..b58399789 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -10,7 +10,7 @@ tag_name = version/{new_version} [bumpversion:part:release] optional_value = stable first_value = beta -values = +values = alpha beta stable @@ -27,33 +27,4 @@ values = [bumpversion:file:passbook/__init__.py] -[bumpversion:file:passbook/api/__init__.py] - -[bumpversion:file:passbook/core/__init__.py] - -[bumpversion:file:passbook/admin/__init__.py] - -[bumpversion:file:passbook/captcha_factor/__init__.py] - -[bumpversion:file:passbook/oauth_client/__init__.py] - -[bumpversion:file:passbook/ldap/__init__.py] - -[bumpversion:file:passbook/lib/__init__.py] - -[bumpversion:file:passbook/hibp_policy/__init__.py] - -[bumpversion:file:passbook/password_expiry_policy/__init__.py] - -[bumpversion:file:passbook/saml_idp/__init__.py] - -[bumpversion:file:passbook/audit/__init__.py] - -[bumpversion:file:passbook/oauth_provider/__init__.py] - -[bumpversion:file:passbook/otp/__init__.py] - -[bumpversion:file:passbook/app_gw/__init__.py] - -[bumpversion:file:passbook/suspicious_policy/__init__.py] - +[bumpversion:file:passbook/core/nginx.conf] diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd9ad4e2d..ea6aec53e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,11 @@ # Global Variables stages: - - build-buildimage + - build-base-image + - build-dev-image - test - build - - docs - - deploy -image: docker.beryju.org/passbook/build-base:latest -services: - - postgres:latest - - redis:latest + - package +image: docker.beryju.org/passbook/dev:latest variables: POSTGRES_DB: passbook @@ -16,51 +13,78 @@ variables: POSTGRES_PASSWORD: "EK-5jnKfjrGRm<77" before_script: - # Ensure all dependencies are installed, even those not included in passbook/build-base + # Ensure all dependencies are installed, even those not included in passbook/dev + - pip install -r requirements.txt - pip install -r requirements-dev.txt -create-build-image: +create-base-image: image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] before_script: - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json script: - - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.build-base --destination docker.beryju.org/passbook/build-base:latest --destination docker.beryju.org/passbook/build-base:0.2.6-beta - stage: build-buildimage + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.base --destination docker.beryju.org/passbook/base:latest --destination docker.beryju.org/passbook/base:0.2.6-beta + stage: build-base-image only: refs: - tags - /^version/.*$/ +build-dev-image: + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + before_script: + - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json + script: + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.dev --destination docker.beryju.org/passbook/dev:latest --destination docker.beryju.org/passbook/dev:0.2.6-beta + stage: build-dev-image + only: + refs: + - tags + - /^version/.*$/ + + isort: script: - isort -c -sg env stage: test + services: + - postgres:latest + - redis:latest migrations: script: - python manage.py migrate stage: test + services: + - postgres:latest + - redis:latest prospector: script: - prospector stage: test + services: + - postgres:latest + - redis:latest pylint: script: - - pylint passbook + - pylint p2 stage: test + services: + - postgres:latest + - redis:latest coverage: script: - - python manage.py collectstatic --no-input - coverage run manage.py test - coverage report + - coverage html stage: test -bandit: - script: - - bandit -r passbook - stage: test + services: + - postgres:latest + - redis:latest -package-docker: +package-passbook-server: image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] @@ -72,42 +96,37 @@ package-docker: only: - tags - /^version/.*$/ -package-helm: +build-passbook-static: stage: build + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + before_script: + - echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json script: + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile.static --destination docker.beryju.org/passbook/static:latest --destination docker.beryju.org/passbook/static:0.2.6-beta + only: + - tags + - /^version/.*$/ + # running collectstatic fully initialises django, hence we need that databases + services: + - postgres:latest + - redis:latest + +package-helm: + image: debian:stretch-slim + stage: package + before_script: + - apt update && apt install -y curl - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash + script: - helm init --client-only + - helm dependency build helm/passbook - helm package helm/passbook artifacts: paths: - passbook-*.tgz - expire_in: 2 days + expire_in: 1 week only: - tags - /^version/.*$/ - -package-client-package-allauth: - script: - - cd client-packages/allauth - - python setup.py sdist - - twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD dist/* - stage: build - only: - refs: - - tags - - /^version/.*$/ - changes: - - client-packages/allauth/** - -package-client-package-sentry: - script: - - cd client-packages/sentry-auth-passbook - - python setup.py sdist - - twine upload --username $TWINE_USERNAME --password $TWINE_PASSWORD dist/* - stage: build - only: - refs: - - tags - - /^version/.*$/ - changes: - - client-packages/sentry-auth-passbook/** diff --git a/Dockerfile b/Dockerfile index 0647936bf..0952e826b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,8 @@ -FROM python:3.6-slim-stretch as build +FROM docker.beryju.org/passbook/base:latest COPY ./passbook/ /app/passbook COPY ./manage.py /app/ -COPY ./requirements.txt /app/ - -WORKDIR /app/ - -RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \ - mkdir /app/static/ && \ - pip install -r requirements.txt && \ - pip install psycopg2 && \ - ./manage.py collectstatic --no-input && \ - apt-get remove --purge -y build-essential && \ - apt-get autoremove --purge -y - -FROM python:3.6-slim-stretch - -COPY ./passbook/ /app/passbook -COPY ./manage.py /app/ -COPY ./requirements.txt /app/ -COPY --from=build /app/static /app/static/ - -WORKDIR /app/ - -RUN apt-get update && apt-get install build-essential libssl-dev libffi-dev libpq-dev -y && \ - pip install -r requirements.txt && \ - pip install psycopg2 && \ - adduser --system --home /app/ passbook && \ - chown -R passbook /app/ && \ - apt-get remove --purge -y build-essential && \ - apt-get autoremove --purge -y USER passbook + +WORKDIR /app/ diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 000000000..b258bee04 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,11 @@ +FROM python:3.7-alpine + +COPY ./requirements.txt /app/ + +WORKDIR /app/ + +RUN apk update && \ + apk add --no-cache openssl-dev build-base libxml2-dev libxslt-dev libffi-dev gcc musl-dev libgcc zlib-dev postgresql-dev && \ + pip install -r /app/requirements.txt --no-cache-dir && \ + adduser -S passbook && \ + chown -R passbook /app diff --git a/Dockerfile.build-base b/Dockerfile.build-base deleted file mode 100644 index 6698a46da..000000000 --- a/Dockerfile.build-base +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.6 - -COPY ./passbook/ /app/passbook -COPY ./client-packages/ /app/client-packages -COPY ./requirements.txt /app/ -COPY ./requirements-dev.txt /app/ - -WORKDIR /app/ - -RUN apt-get update && apt-get install libssl-dev libffi-dev libpq-dev -y && \ - pip install -U -r requirements-dev.txt && \ - rm -rf /app/* diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000..2174bf229 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,5 @@ +FROM docker.beryju.org/passbook/base:latest + +COPY ./requirements-dev.txt /app/ + +RUN pip install -r /app/requirements-dev.txt --no-cache-dir diff --git a/Dockerfile.static b/Dockerfile.static new file mode 100644 index 000000000..e7a0409fb --- /dev/null +++ b/Dockerfile.static @@ -0,0 +1,14 @@ +FROM docker.beryju.org/passbook/dev:latest as static-build + +COPY ./passbook/ /app/passbook +COPY ./manage.py /app/ +COPY ./requirements.txt /app/ + +WORKDIR /app/ + +RUN ./manage.py collectstatic --no-input + +FROM nginx:latest + +COPY --from=static-build /app/static /static/_/static/ +COPY ./passbook/core/nginx.conf /etc/nginx/nginx.conf diff --git a/requirements-dev.txt b/requirements-dev.txt index 85d364c25..5dbe33140 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,4 @@ -r requirements.txt --r client-packages/allauth/requirements.txt coverage isort astroid==2.0.4 @@ -14,3 +13,4 @@ autopep8 bandit bumpversion twine +grpcio-tools diff --git a/requirements.txt b/requirements.txt index 8f2ab6fd8..ef59f4925 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,51 @@ --r passbook/root/requirements.txt --r passbook/oauth_client/requirements.txt --r passbook/ldap/requirements.txt --r passbook/saml_idp/requirements.txt --r passbook/otp/requirements.txt --r passbook/oauth_provider/requirements.txt --r passbook/captcha_factor/requirements.txt --r passbook/admin/requirements.txt --r passbook/api/requirements.txt --r passbook/app_gw/requirements.txt --r passbook/oidc_provider/requirements.txt +# Root requirements +celery +colorlog +cherrypy +django-ipware +django-model-utils +django-redis +django>=2.0 +idna<2.8,>=2.5 +markdown +psycopg2 +PyYAML +sentry-sdk +pip +urllib3<1.25,>=1.21.1 +grpcio +grpcio-reflection +protobuf +# OAuth Client +requests_oauthlib>=0.4.2 +oauthlib>=2.0.6 +# LDAP Client +ldap3 +# SAML IDP +beautifulsoup4>=4.6.0 +lxml>=3.8.0 +signxml +defusedxml +PyCryptodome +# OTP +django_otp +qrcode +# OAuth Provider +django-oauth-toolkit +django-cors-middleware +# ReCaptcha +django-recaptcha +# API +drf_yasg +djangorestframework==3.9.4 +django-filters +# AppGW +django-revproxy +urllib3[secure] +channels +service_identity +websocket-client +daphne<2.3.0 +asgiref~=2.3 +# OIDC Provider +django-oidc-provider