add basic dockerization to devicehub dpp

2023-09-19 18:03:23 +02:00 · 2023-09-19 18:03:23 +02:00 · 907bf2dba0
parent 0b70f42daa
commit 907bf2dba0
9 changed files with 308 additions and 1 deletions
--- a/.gitignore
+++ b/.gitignore
@ -136,3 +136,6 @@ examples/create-db2.sh
 package-lock.json
 snapshots/
 modules/
+
+# emacs
+*~
--- a/30
+++ b/30
@ -0,0 +1,30 @@
+project := dkr-dsg.ac.upc.edu/devicehub
+
+branch := `git branch --show-current`
+commit := `git log -1 --format=%h`
+tag := ${branch}__${commit}
+
+# docker images
+devicehub_image := ${project}/devicehub:${tag}
+postgres_image := ${project}/postgres:${tag}
+
+# 2. Create a virtual environment.
+docker_build:
+	docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} .
+	# DEBUG
+	#docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . --progress=plain --no-cache
+
+	docker build -f docker/postgres.Dockerfile -t ${postgres_image} .
+	# DEBUG
+	#docker build -f docker/postgres.Dockerfile -t ${postgres_image} . --progress=plain --no-cache
+
+docker_publish:
+	docker push ${devicehub_image}
+
+.PHONY: docker
+docker:
+	$(MAKE) docker_build
+	#$(MAKE) docker_publish
+	@printf "\nimage: ${devicehub_image}\n"
+	@printf "\nimage: ${postgres_image}\n"
+	@printf "\ndocker images built and published\n"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,47 @@
+version: "3.9"
+services:
+
+  devicehub:
+    init: true
+    # TODO
+    image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp__0fb4fa5b
+    #build .
+    environment:
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_HOST=postgres
+      - DB_DATABASE=${DB_DATABASE}
+      - HOST=${HOST}
+      - EMAIL_DEMO=${EMAIL_DEMO}
+      - PASSWORD_DEMO=${PASSWORD_DEMO}
+      - JWT_PASS=${JWT_PASS}
+      - SECRET_KEY=${SECRET_KEY}
+      - API_DLT=${API_DLT}
+      - API_RESOLVER=${API_RESOLVER}
+      - API_DLT_TOKEN=${API_DLT_TOKEN}
+      - DEVICEHUB_HOST=${DEVICEHUB_HOST}
+      - ID_FEDERATED=${ID_FEDERATED}
+      - URL_MANUALS=${URL_MANUALS}
+    ports:
+      - 5000:5000
+    volumes:
+      - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
+
+  postgres:
+    image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp__0fb4fa5b
+    # 4. To create the database.
+    # 5. Give permissions to the corresponding users in the database.
+    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
+    environment:
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_DB=${DB_DATABASE}
+    ports:
+      - 5432:5432
+    # TODO persistence
+    #volumes:
+    #  - pg_data:/var/lib/postgresql/data
+
+
+  # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
+  #nginx
--- a/docker/devicehub.Dockerfile
+++ b/docker/devicehub.Dockerfile
@ -0,0 +1,30 @@
+FROM debian:bullseye-slim
+
+RUN apt update && apt-get install --no-install-recommends -y \
+    python3-minimal \
+    python3-pip \
+    python-is-python3 \
+    python3-psycopg2 \
+    python3-dev \
+    libpq-dev \
+    build-essential \
+    libpangocairo-1.0-0 \
+    curl \
+    jq \
+    time \
+    netcat
+
+WORKDIR /opt/devicehub
+
+# this is exactly the same as examples/pip_install.sh except the last command
+#   to improve the docker layer builds, it has been separated
+RUN pip install --upgrade pip
+RUN pip install alembic==1.8.1    anytree==2.8.0    apispec==0.39.0    atomicwrites==1.4.0    blinker==1.5    boltons==23.0.0    cairocffi==1.4.0            cairosvg==2.5.2    certifi==2022.9.24    cffi==1.15.1            charset-normalizer==2.0.12    click==6.7    click-spinner==0.1.8    colorama==0.3.9    colour==0.1.5    cssselect2==0.7.0            defusedxml==0.7.1            et-xmlfile==1.1.0            flask==1.0.2                        flask-cors==3.0.10    flask-login==0.5.0    flask-sqlalchemy==2.5.1    flask-weasyprint==0.4    flask-wtf==1.0.0    hashids==1.2.0    html5lib==1.1    idna==3.4    inflection==0.5.1    itsdangerous==2.0.1            jinja2==3.0.3    mako==1.2.3    markupsafe==2.1.1                marshmallow==3.0.0b11                marshmallow-enum==1.4.1    more-itertools==8.12.0    numpy==1.22.0            odfpy==1.4.1    openpyxl==3.0.10    pandas==1.3.5    passlib==1.7.1    phonenumbers==8.9.11    pillow==9.2.0    pint==0.9    psycopg2-binary==2.8.3    py-dmidecode==0.1.0    pycparser==2.21    pyjwt==2.4.0    pyphen==0.13.0    python-dateutil==2.7.3            python-decouple==3.3    python-dotenv==0.14.0    python-editor==1.0.4    python-stdnum==1.9    pytz==2022.2.1    pyyaml==5.4            requests==2.27.1                requests-mock==1.5.2    requests-toolbelt==0.9.1    six==1.16.0                            sortedcontainers==2.1.0    sqlalchemy==1.3.24                    sqlalchemy-citext==1.3.post0    sqlalchemy-utils==0.33.11    tinycss2==1.1.1                tqdm==4.32.2    urllib3==1.26.12    weasyprint==44    webargs==5.5.3    webencodings==0.5.1                werkzeug==2.0.3            wtforms==3.0.1    xlrd==2.0.1   cryptography==39.0.1 Authlib==1.2.1 gunicorn==21.2.0
+
+RUN pip install -i https://test.pypi.org/simple/ ereuseapitest==0.0.8
+
+COPY . .
+RUN pip install -e .
+
+COPY docker/devicehub.entrypoint.sh .
+ENTRYPOINT sh ./devicehub.entrypoint.sh
--- a/docker/devicehub.Dockerfile.dockerignore
+++ b/docker/devicehub.Dockerfile.dockerignore
@ -0,0 +1,12 @@
+.git
+.env
+# TODO need to comment it to copy the entrypoint
+#docker
+Makefile
+
+# Emacs backup files
+*~
+.\#*
+# Vim swap files
+*.swp
+*.swo
--- a/docker/devicehub.entrypoint.sh
+++ b/docker/devicehub.entrypoint.sh
@ -0,0 +1,154 @@
+#!/bin/sh
+
+set -e
+set -u
+# DEBUG
+set -x
+
+# 3. Generate an environment .env file.
+gen_env_vars() {
+        # generate config using env vars from docker
+        cat > .env <<END
+DB_USER=${DB_USER}
+DB_PASSWORD=${DB_PASSWORD}
+DB_HOST=${DB_HOST}
+DB_DATABASE=${DB_DATABASE}
+SCHEMA=dbtest
+DB_SCHEMA=dbtest
+HOST=${HOST}
+EMAIL_DEMO=${EMAIL_DEMO}
+PASSWORD_DEMO=${PASSWORD_DEMO}
+JWT_PASS=${JWT_PASS}
+SECRET_KEY=${SECRET_KEY}
+
+API_DLT=${API_DLT}
+API_RESOLVER=${API_RESOLVER}
+API_DLT_TOKEN=${API_DLT_TOKEN}
+ID_FEDERATED=${ID_FEDERATED}
+
+OAUTH2_JWT_ENABLED=True
+OAUTH2_JWT_ISS=https://authlib.org
+OAUTH2_JWT_KEY=secret-key
+OAUTH2_JWT_ALG=HS256
+
+URL_MANUALS=${URL_MANUALS}
+END
+}
+
+wait_for_postgres() {
+        # old one was
+        #sleep 4
+
+        default_postgres_port=5432
+        # thanks https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
+        while ! nc -z ${DB_HOST} ${default_postgres_port}; do
+                sleep 0.5
+        done
+}
+
+init_data() {
+
+        # 7. Run alembic of the project.
+        alembic -x inventory=dbtest upgrade head
+        # 8. Running alembic from oidc module.y
+        cd ereuse_devicehub/modules/oidc
+        alembic -x inventory=dbtest upgrade head
+        cd -
+        # 9. Running alembic from dpp module.
+        cd ereuse_devicehub/modules/dpp/
+        alembic -x inventory=dbtest upgrade head
+        cd -
+
+        # 11. Generate a minimal data structure.
+        #   TODO it has some errors (?)
+        flask initdata || true
+}
+
+big_error() {
+        local message="${@}"
+        echo "###############################################" >&2
+        echo "# ERROR: ${message}" >&2
+        echo "###############################################" >&2
+        exit 1
+}
+
+handle_federated_id() {
+
+        # devicehub host and id federated checker
+
+        EXPECTED_ID_FEDERATED="$(curl -s "${API_RESOLVER}/getAll" \
+                | jq -r '.url | to_entries | .[] | select(.value == "'"${DEVICEHUB_HOST}"'") | .key' \
+                | head -n 1)"
+
+        # if is a new DEVICEHUB_HOST, then register it
+        if [ -z "${EXPECTED_ID_FEDERATED}" ]; then
+                # TODO better docker compose run command
+                cmd="docker compose run --entrypoint= devicehub flask dlt_insert_members ${DEVICEHUB_HOST}"
+                big_error "No FEDERATED ID maybe you should run \`${cmd}\`"
+        fi
+
+        # if not new DEVICEHUB_HOST, then check consistency
+
+        # if there is already an ID in the DLT, it should match with my internal ID
+        if [ ! "${EXPECTED_ID_FEDERATED}" = "${ID_FEDERATED}" ]; then
+
+                big_error "ID_FEDERATED should be ${EXPECTED_ID_FEDERATED} instead of ${ID_FEDERATED}"
+        fi
+
+        # not needed, but reserved
+        # EXPECTED_DEVICEHUB_HOST="$(curl -s "${API_RESOLVER}/getAll" \
+        #         | jq -r '.url | to_entries | .[] | select(.key == "'"${ID_FEDERATED}"'") | .value' \
+        #         | head -n 1)"
+        # if [ ! "${EXPECTED_DEVICEHUB_HOST}" = "${DEVICEHUB_HOST}" ]; then
+        #         big_error "ERROR: DEVICEHUB_HOST should be ${EXPECTED_DEVICEHUB_HOST} instead of ${DEVICEHUB_HOST}"
+        # fi
+
+}
+
+main() {
+
+        gen_env_vars
+
+        wait_for_postgres
+
+        init_flagfile='/container_initialized'
+        if [ ! -f "${init_flagfile}" ]; then
+
+                # 7, 8, 9, 11
+                init_data
+
+                # 12. Add a new server to the 'api resolver'
+                handle_federated_id
+
+                # 13. Do a rsync api resolve
+                flask dlt_rsync_members
+
+                # 14. Register a new user to the DLT
+                flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
+
+                # non DL user (only for the inventory)
+                #   flask adduser user2@dhub.com ${PASSWORD_DEMO}
+
+                # # 15. Add inventory snapshots for user "${EMAIL_DEMO}".
+                cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
+                /usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+
+                # # 16.
+                flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+
+                # remain next command as the last operation for this if conditional
+                touch "${init_flagfile}"
+        fi
+
+        # 17. Use gunicorn
+        #   thanks https://akira3030.github.io/formacion/articulos/python-flask-gunicorn-docker.html
+        # TODO meanwhile no nginx (step 19), gunicorn cannot serve static files, then we prefer development server
+        #gunicorn --access-logfile - --error-logfile - --workers 4 -b :5000 app:app
+        #   alternative: run development server
+        flask run --host=0.0.0.0 --port 5000
+
+        # DEBUG
+        #sleep infinity
+}
+
+main "${@}"
--- a/docker/postgres.Dockerfile
+++ b/docker/postgres.Dockerfile
@ -0,0 +1,8 @@
+FROM postgres:15.4-bookworm
+# this is the latest in 2023-09-14_13-01-38
+#FROM postgres:latest
+
+# Add a SQL script that will be executed upon container startup
+COPY docker/postgres.setupdb.sql /docker-entrypoint-initdb.d/
+
+EXPOSE 5432
--- a/docker/postgres.setupdb.sql
+++ b/docker/postgres.setupdb.sql
@ -0,0 +1,5 @@
+-- 6. Create the necessary extensions.
+CREATE EXTENSION pgcrypto SCHEMA public;
+CREATE EXTENSION ltree SCHEMA public;
+CREATE EXTENSION citext SCHEMA public;
+CREATE EXTENSION pg_trgm SCHEMA public;
--- a/examples/env.example
+++ b/examples/env.example
@ -5,6 +5,24 @@ DB_DATABASE='devicehub'
 API_DLT='http://$IP_API_DLT'
 API_DLT_TOKEN=$TOKEN
 API_RESOLVER='http://$IP_API_RESOLVER'
-ID_FEDERATED='DH12'
+# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
+ID_FEDERATED='$ID'
 URL_MANUALS='http://$IP_MANUALS'

+DEVICEHUB_HOST='http://devicehub.example.com'
+HOST='localhost'
+
+SCHEMA='dbtest'
+DB_SCHEMA='dbtest'
+
+EMAIL_DEMO='user@example.org'
+PASSWORD_DEMO='changeme'
+
+JWT_PASS='changeme'
+SECRET_KEY='changeme'
+
+# important to import snapshots (step 15)
+#   rel path starts with ./
+#SNAPSHOTS_PATH='./relpath/to/snapshots'
+#   full path starts with /
+SNAPSHOTS_PATH='/fullpath/to/snapshots'