add basic dockerization to devicehub dpp
This commit is contained in:
pedro
parent
0b70f42daa
commit
907bf2dba0
|
|
@ -136,3 +136,6 @@ examples/create-db2.sh
|
||||||
package-lock.json
|
package-lock.json
|
||||||
snapshots/
|
snapshots/
|
||||||
modules/
|
modules/
|
||||||
|
|
||||||
|
# emacs
|
||||||
|
*~
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
project := dkr-dsg.ac.upc.edu/devicehub
|
||||||
|
|
||||||
|
branch := `git branch --show-current`
|
||||||
|
commit := `git log -1 --format=%h`
|
||||||
|
tag := ${branch}__${commit}
|
||||||
|
|
||||||
|
# docker images
|
||||||
|
devicehub_image := ${project}/devicehub:${tag}
|
||||||
|
postgres_image := ${project}/postgres:${tag}
|
||||||
|
|
||||||
|
# 2. Create a virtual environment.
|
||||||
|
docker_build:
|
||||||
|
docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} .
|
||||||
|
# DEBUG
|
||||||
|
#docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . --progress=plain --no-cache
|
||||||
|
|
||||||
|
docker build -f docker/postgres.Dockerfile -t ${postgres_image} .
|
||||||
|
# DEBUG
|
||||||
|
#docker build -f docker/postgres.Dockerfile -t ${postgres_image} . --progress=plain --no-cache
|
||||||
|
|
||||||
|
docker_publish:
|
||||||
|
docker push ${devicehub_image}
|
||||||
|
|
||||||
|
.PHONY: docker
|
||||||
|
docker:
|
||||||
|
$(MAKE) docker_build
|
||||||
|
#$(MAKE) docker_publish
|
||||||
|
@printf "\nimage: ${devicehub_image}\n"
|
||||||
|
@printf "\nimage: ${postgres_image}\n"
|
||||||
|
@printf "\ndocker images built and published\n"
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
version: "3.9"
|
||||||
|
services:
|
||||||
|
|
||||||
|
devicehub:
|
||||||
|
init: true
|
||||||
|
# TODO
|
||||||
|
image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp__0fb4fa5b
|
||||||
|
#build .
|
||||||
|
environment:
|
||||||
|
- DB_USER=${DB_USER}
|
||||||
|
- DB_PASSWORD=${DB_PASSWORD}
|
||||||
|
- DB_HOST=postgres
|
||||||
|
- DB_DATABASE=${DB_DATABASE}
|
||||||
|
- HOST=${HOST}
|
||||||
|
- EMAIL_DEMO=${EMAIL_DEMO}
|
||||||
|
- PASSWORD_DEMO=${PASSWORD_DEMO}
|
||||||
|
- JWT_PASS=${JWT_PASS}
|
||||||
|
- SECRET_KEY=${SECRET_KEY}
|
||||||
|
- API_DLT=${API_DLT}
|
||||||
|
- API_RESOLVER=${API_RESOLVER}
|
||||||
|
- API_DLT_TOKEN=${API_DLT_TOKEN}
|
||||||
|
- DEVICEHUB_HOST=${DEVICEHUB_HOST}
|
||||||
|
- ID_FEDERATED=${ID_FEDERATED}
|
||||||
|
- URL_MANUALS=${URL_MANUALS}
|
||||||
|
ports:
|
||||||
|
- 5000:5000
|
||||||
|
volumes:
|
||||||
|
- ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp__0fb4fa5b
|
||||||
|
# 4. To create the database.
|
||||||
|
# 5. Give permissions to the corresponding users in the database.
|
||||||
|
# extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
|
||||||
|
environment:
|
||||||
|
- POSTGRES_PASSWORD=${DB_PASSWORD}
|
||||||
|
- POSTGRES_USER=${DB_USER}
|
||||||
|
- POSTGRES_DB=${DB_DATABASE}
|
||||||
|
ports:
|
||||||
|
- 5432:5432
|
||||||
|
# TODO persistence
|
||||||
|
#volumes:
|
||||||
|
# - pg_data:/var/lib/postgresql/data
|
||||||
|
|
||||||
|
|
||||||
|
# TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
|
||||||
|
#nginx
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
FROM debian:bullseye-slim
|
||||||
|
|
||||||
|
RUN apt update && apt-get install --no-install-recommends -y \
|
||||||
|
python3-minimal \
|
||||||
|
python3-pip \
|
||||||
|
python-is-python3 \
|
||||||
|
python3-psycopg2 \
|
||||||
|
python3-dev \
|
||||||
|
libpq-dev \
|
||||||
|
build-essential \
|
||||||
|
libpangocairo-1.0-0 \
|
||||||
|
curl \
|
||||||
|
jq \
|
||||||
|
time \
|
||||||
|
netcat
|
||||||
|
|
||||||
|
WORKDIR /opt/devicehub
|
||||||
|
|
||||||
|
# this is exactly the same as examples/pip_install.sh except the last command
|
||||||
|
# to improve the docker layer builds, it has been separated
|
||||||
|
RUN pip install --upgrade pip
|
||||||
|
RUN pip install alembic==1.8.1 anytree==2.8.0 apispec==0.39.0 atomicwrites==1.4.0 blinker==1.5 boltons==23.0.0 cairocffi==1.4.0 cairosvg==2.5.2 certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.0.12 click==6.7 click-spinner==0.1.8 colorama==0.3.9 colour==0.1.5 cssselect2==0.7.0 defusedxml==0.7.1 et-xmlfile==1.1.0 flask==1.0.2 flask-cors==3.0.10 flask-login==0.5.0 flask-sqlalchemy==2.5.1 flask-weasyprint==0.4 flask-wtf==1.0.0 hashids==1.2.0 html5lib==1.1 idna==3.4 inflection==0.5.1 itsdangerous==2.0.1 jinja2==3.0.3 mako==1.2.3 markupsafe==2.1.1 marshmallow==3.0.0b11 marshmallow-enum==1.4.1 more-itertools==8.12.0 numpy==1.22.0 odfpy==1.4.1 openpyxl==3.0.10 pandas==1.3.5 passlib==1.7.1 phonenumbers==8.9.11 pillow==9.2.0 pint==0.9 psycopg2-binary==2.8.3 py-dmidecode==0.1.0 pycparser==2.21 pyjwt==2.4.0 pyphen==0.13.0 python-dateutil==2.7.3 python-decouple==3.3 python-dotenv==0.14.0 python-editor==1.0.4 python-stdnum==1.9 pytz==2022.2.1 pyyaml==5.4 requests==2.27.1 requests-mock==1.5.2 requests-toolbelt==0.9.1 six==1.16.0 sortedcontainers==2.1.0 sqlalchemy==1.3.24 sqlalchemy-citext==1.3.post0 sqlalchemy-utils==0.33.11 tinycss2==1.1.1 tqdm==4.32.2 urllib3==1.26.12 weasyprint==44 webargs==5.5.3 webencodings==0.5.1 werkzeug==2.0.3 wtforms==3.0.1 xlrd==2.0.1 cryptography==39.0.1 Authlib==1.2.1 gunicorn==21.2.0
|
||||||
|
|
||||||
|
RUN pip install -i https://test.pypi.org/simple/ ereuseapitest==0.0.8
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
RUN pip install -e .
|
||||||
|
|
||||||
|
COPY docker/devicehub.entrypoint.sh .
|
||||||
|
ENTRYPOINT sh ./devicehub.entrypoint.sh
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
.git
|
||||||
|
.env
|
||||||
|
# TODO need to comment it to copy the entrypoint
|
||||||
|
#docker
|
||||||
|
Makefile
|
||||||
|
|
||||||
|
# Emacs backup files
|
||||||
|
*~
|
||||||
|
.\#*
|
||||||
|
# Vim swap files
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
|
@ -0,0 +1,154 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -u
|
||||||
|
# DEBUG
|
||||||
|
set -x
|
||||||
|
|
||||||
|
# 3. Generate an environment .env file.
|
||||||
|
gen_env_vars() {
|
||||||
|
# generate config using env vars from docker
|
||||||
|
cat > .env <<END
|
||||||
|
DB_USER=${DB_USER}
|
||||||
|
DB_PASSWORD=${DB_PASSWORD}
|
||||||
|
DB_HOST=${DB_HOST}
|
||||||
|
DB_DATABASE=${DB_DATABASE}
|
||||||
|
SCHEMA=dbtest
|
||||||
|
DB_SCHEMA=dbtest
|
||||||
|
HOST=${HOST}
|
||||||
|
EMAIL_DEMO=${EMAIL_DEMO}
|
||||||
|
PASSWORD_DEMO=${PASSWORD_DEMO}
|
||||||
|
JWT_PASS=${JWT_PASS}
|
||||||
|
SECRET_KEY=${SECRET_KEY}
|
||||||
|
|
||||||
|
API_DLT=${API_DLT}
|
||||||
|
API_RESOLVER=${API_RESOLVER}
|
||||||
|
API_DLT_TOKEN=${API_DLT_TOKEN}
|
||||||
|
ID_FEDERATED=${ID_FEDERATED}
|
||||||
|
|
||||||
|
OAUTH2_JWT_ENABLED=True
|
||||||
|
OAUTH2_JWT_ISS=https://authlib.org
|
||||||
|
OAUTH2_JWT_KEY=secret-key
|
||||||
|
OAUTH2_JWT_ALG=HS256
|
||||||
|
|
||||||
|
URL_MANUALS=${URL_MANUALS}
|
||||||
|
END
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_postgres() {
|
||||||
|
# old one was
|
||||||
|
#sleep 4
|
||||||
|
|
||||||
|
default_postgres_port=5432
|
||||||
|
# thanks https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
|
||||||
|
while ! nc -z ${DB_HOST} ${default_postgres_port}; do
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
init_data() {
|
||||||
|
|
||||||
|
# 7. Run alembic of the project.
|
||||||
|
alembic -x inventory=dbtest upgrade head
|
||||||
|
# 8. Running alembic from oidc module.y
|
||||||
|
cd ereuse_devicehub/modules/oidc
|
||||||
|
alembic -x inventory=dbtest upgrade head
|
||||||
|
cd -
|
||||||
|
# 9. Running alembic from dpp module.
|
||||||
|
cd ereuse_devicehub/modules/dpp/
|
||||||
|
alembic -x inventory=dbtest upgrade head
|
||||||
|
cd -
|
||||||
|
|
||||||
|
# 11. Generate a minimal data structure.
|
||||||
|
# TODO it has some errors (?)
|
||||||
|
flask initdata || true
|
||||||
|
}
|
||||||
|
|
||||||
|
big_error() {
|
||||||
|
local message="${@}"
|
||||||
|
echo "###############################################" >&2
|
||||||
|
echo "# ERROR: ${message}" >&2
|
||||||
|
echo "###############################################" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
handle_federated_id() {
|
||||||
|
|
||||||
|
# devicehub host and id federated checker
|
||||||
|
|
||||||
|
EXPECTED_ID_FEDERATED="$(curl -s "${API_RESOLVER}/getAll" \
|
||||||
|
| jq -r '.url | to_entries | .[] | select(.value == "'"${DEVICEHUB_HOST}"'") | .key' \
|
||||||
|
| head -n 1)"
|
||||||
|
|
||||||
|
# if is a new DEVICEHUB_HOST, then register it
|
||||||
|
if [ -z "${EXPECTED_ID_FEDERATED}" ]; then
|
||||||
|
# TODO better docker compose run command
|
||||||
|
cmd="docker compose run --entrypoint= devicehub flask dlt_insert_members ${DEVICEHUB_HOST}"
|
||||||
|
big_error "No FEDERATED ID maybe you should run \`${cmd}\`"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# if not new DEVICEHUB_HOST, then check consistency
|
||||||
|
|
||||||
|
# if there is already an ID in the DLT, it should match with my internal ID
|
||||||
|
if [ ! "${EXPECTED_ID_FEDERATED}" = "${ID_FEDERATED}" ]; then
|
||||||
|
|
||||||
|
big_error "ID_FEDERATED should be ${EXPECTED_ID_FEDERATED} instead of ${ID_FEDERATED}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# not needed, but reserved
|
||||||
|
# EXPECTED_DEVICEHUB_HOST="$(curl -s "${API_RESOLVER}/getAll" \
|
||||||
|
# | jq -r '.url | to_entries | .[] | select(.key == "'"${ID_FEDERATED}"'") | .value' \
|
||||||
|
# | head -n 1)"
|
||||||
|
# if [ ! "${EXPECTED_DEVICEHUB_HOST}" = "${DEVICEHUB_HOST}" ]; then
|
||||||
|
# big_error "ERROR: DEVICEHUB_HOST should be ${EXPECTED_DEVICEHUB_HOST} instead of ${DEVICEHUB_HOST}"
|
||||||
|
# fi
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
|
||||||
|
gen_env_vars
|
||||||
|
|
||||||
|
wait_for_postgres
|
||||||
|
|
||||||
|
init_flagfile='/container_initialized'
|
||||||
|
if [ ! -f "${init_flagfile}" ]; then
|
||||||
|
|
||||||
|
# 7, 8, 9, 11
|
||||||
|
init_data
|
||||||
|
|
||||||
|
# 12. Add a new server to the 'api resolver'
|
||||||
|
handle_federated_id
|
||||||
|
|
||||||
|
# 13. Do a rsync api resolve
|
||||||
|
flask dlt_rsync_members
|
||||||
|
|
||||||
|
# 14. Register a new user to the DLT
|
||||||
|
flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
|
||||||
|
|
||||||
|
# non DL user (only for the inventory)
|
||||||
|
# flask adduser user2@dhub.com ${PASSWORD_DEMO}
|
||||||
|
|
||||||
|
# # 15. Add inventory snapshots for user "${EMAIL_DEMO}".
|
||||||
|
cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
|
||||||
|
/usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
|
||||||
|
|
||||||
|
# # 16.
|
||||||
|
flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
|
||||||
|
|
||||||
|
# remain next command as the last operation for this if conditional
|
||||||
|
touch "${init_flagfile}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 17. Use gunicorn
|
||||||
|
# thanks https://akira3030.github.io/formacion/articulos/python-flask-gunicorn-docker.html
|
||||||
|
# TODO meanwhile no nginx (step 19), gunicorn cannot serve static files, then we prefer development server
|
||||||
|
#gunicorn --access-logfile - --error-logfile - --workers 4 -b :5000 app:app
|
||||||
|
# alternative: run development server
|
||||||
|
flask run --host=0.0.0.0 --port 5000
|
||||||
|
|
||||||
|
# DEBUG
|
||||||
|
#sleep infinity
|
||||||
|
}
|
||||||
|
|
||||||
|
main "${@}"
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
FROM postgres:15.4-bookworm
|
||||||
|
# this is the latest in 2023-09-14_13-01-38
|
||||||
|
#FROM postgres:latest
|
||||||
|
|
||||||
|
# Add a SQL script that will be executed upon container startup
|
||||||
|
COPY docker/postgres.setupdb.sql /docker-entrypoint-initdb.d/
|
||||||
|
|
||||||
|
EXPOSE 5432
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
-- 6. Create the necessary extensions.
|
||||||
|
CREATE EXTENSION pgcrypto SCHEMA public;
|
||||||
|
CREATE EXTENSION ltree SCHEMA public;
|
||||||
|
CREATE EXTENSION citext SCHEMA public;
|
||||||
|
CREATE EXTENSION pg_trgm SCHEMA public;
|
||||||
|
|
@ -5,6 +5,24 @@ DB_DATABASE='devicehub'
|
||||||
API_DLT='http://$IP_API_DLT'
|
API_DLT='http://$IP_API_DLT'
|
||||||
API_DLT_TOKEN=$TOKEN
|
API_DLT_TOKEN=$TOKEN
|
||||||
API_RESOLVER='http://$IP_API_RESOLVER'
|
API_RESOLVER='http://$IP_API_RESOLVER'
|
||||||
ID_FEDERATED='DH12'
|
# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
|
||||||
|
ID_FEDERATED='$ID'
|
||||||
URL_MANUALS='http://$IP_MANUALS'
|
URL_MANUALS='http://$IP_MANUALS'
|
||||||
|
|
||||||
|
DEVICEHUB_HOST='http://devicehub.example.com'
|
||||||
|
HOST='localhost'
|
||||||
|
|
||||||
|
SCHEMA='dbtest'
|
||||||
|
DB_SCHEMA='dbtest'
|
||||||
|
|
||||||
|
EMAIL_DEMO='user@example.org'
|
||||||
|
PASSWORD_DEMO='changeme'
|
||||||
|
|
||||||
|
JWT_PASS='changeme'
|
||||||
|
SECRET_KEY='changeme'
|
||||||
|
|
||||||
|
# important to import snapshots (step 15)
|
||||||
|
# rel path starts with ./
|
||||||
|
#SNAPSHOTS_PATH='./relpath/to/snapshots'
|
||||||
|
# full path starts with /
|
||||||
|
SNAPSHOTS_PATH='/fullpath/to/snapshots'
|
||||||
|
|
|
||||||
Reference in New Issue