add basic dockerization to devicehub dpp
This commit is contained in:
pedro
parent
0b70f42daa
commit
907bf2dba0
|
|
@ -136,3 +136,6 @@ examples/create-db2.sh
|
|||
package-lock.json
|
||||
snapshots/
|
||||
modules/
|
||||
|
||||
# emacs
|
||||
*~
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
project := dkr-dsg.ac.upc.edu/devicehub
|
||||
|
||||
branch := `git branch --show-current`
|
||||
commit := `git log -1 --format=%h`
|
||||
tag := ${branch}__${commit}
|
||||
|
||||
# docker images
|
||||
devicehub_image := ${project}/devicehub:${tag}
|
||||
postgres_image := ${project}/postgres:${tag}
|
||||
|
||||
# 2. Create a virtual environment.
|
||||
docker_build:
|
||||
docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} .
|
||||
# DEBUG
|
||||
#docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . --progress=plain --no-cache
|
||||
|
||||
docker build -f docker/postgres.Dockerfile -t ${postgres_image} .
|
||||
# DEBUG
|
||||
#docker build -f docker/postgres.Dockerfile -t ${postgres_image} . --progress=plain --no-cache
|
||||
|
||||
docker_publish:
|
||||
docker push ${devicehub_image}
|
||||
|
||||
.PHONY: docker
|
||||
docker:
|
||||
$(MAKE) docker_build
|
||||
#$(MAKE) docker_publish
|
||||
@printf "\nimage: ${devicehub_image}\n"
|
||||
@printf "\nimage: ${postgres_image}\n"
|
||||
@printf "\ndocker images built and published\n"
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
version: "3.9"
|
||||
services:
|
||||
|
||||
devicehub:
|
||||
init: true
|
||||
# TODO
|
||||
image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp__0fb4fa5b
|
||||
#build .
|
||||
environment:
|
||||
- DB_USER=${DB_USER}
|
||||
- DB_PASSWORD=${DB_PASSWORD}
|
||||
- DB_HOST=postgres
|
||||
- DB_DATABASE=${DB_DATABASE}
|
||||
- HOST=${HOST}
|
||||
- EMAIL_DEMO=${EMAIL_DEMO}
|
||||
- PASSWORD_DEMO=${PASSWORD_DEMO}
|
||||
- JWT_PASS=${JWT_PASS}
|
||||
- SECRET_KEY=${SECRET_KEY}
|
||||
- API_DLT=${API_DLT}
|
||||
- API_RESOLVER=${API_RESOLVER}
|
||||
- API_DLT_TOKEN=${API_DLT_TOKEN}
|
||||
- DEVICEHUB_HOST=${DEVICEHUB_HOST}
|
||||
- ID_FEDERATED=${ID_FEDERATED}
|
||||
- URL_MANUALS=${URL_MANUALS}
|
||||
ports:
|
||||
- 5000:5000
|
||||
volumes:
|
||||
- ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
|
||||
|
||||
postgres:
|
||||
image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp__0fb4fa5b
|
||||
# 4. To create the database.
|
||||
# 5. Give permissions to the corresponding users in the database.
|
||||
# extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
|
||||
environment:
|
||||
- POSTGRES_PASSWORD=${DB_PASSWORD}
|
||||
- POSTGRES_USER=${DB_USER}
|
||||
- POSTGRES_DB=${DB_DATABASE}
|
||||
ports:
|
||||
- 5432:5432
|
||||
# TODO persistence
|
||||
#volumes:
|
||||
# - pg_data:/var/lib/postgresql/data
|
||||
|
||||
|
||||
# TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
|
||||
#nginx
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
FROM debian:bullseye-slim
|
||||
|
||||
RUN apt update && apt-get install --no-install-recommends -y \
|
||||
python3-minimal \
|
||||
python3-pip \
|
||||
python-is-python3 \
|
||||
python3-psycopg2 \
|
||||
python3-dev \
|
||||
libpq-dev \
|
||||
build-essential \
|
||||
libpangocairo-1.0-0 \
|
||||
curl \
|
||||
jq \
|
||||
time \
|
||||
netcat
|
||||
|
||||
WORKDIR /opt/devicehub
|
||||
|
||||
# this is exactly the same as examples/pip_install.sh except the last command
|
||||
# to improve the docker layer builds, it has been separated
|
||||
RUN pip install --upgrade pip
|
||||
RUN pip install alembic==1.8.1 anytree==2.8.0 apispec==0.39.0 atomicwrites==1.4.0 blinker==1.5 boltons==23.0.0 cairocffi==1.4.0 cairosvg==2.5.2 certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.0.12 click==6.7 click-spinner==0.1.8 colorama==0.3.9 colour==0.1.5 cssselect2==0.7.0 defusedxml==0.7.1 et-xmlfile==1.1.0 flask==1.0.2 flask-cors==3.0.10 flask-login==0.5.0 flask-sqlalchemy==2.5.1 flask-weasyprint==0.4 flask-wtf==1.0.0 hashids==1.2.0 html5lib==1.1 idna==3.4 inflection==0.5.1 itsdangerous==2.0.1 jinja2==3.0.3 mako==1.2.3 markupsafe==2.1.1 marshmallow==3.0.0b11 marshmallow-enum==1.4.1 more-itertools==8.12.0 numpy==1.22.0 odfpy==1.4.1 openpyxl==3.0.10 pandas==1.3.5 passlib==1.7.1 phonenumbers==8.9.11 pillow==9.2.0 pint==0.9 psycopg2-binary==2.8.3 py-dmidecode==0.1.0 pycparser==2.21 pyjwt==2.4.0 pyphen==0.13.0 python-dateutil==2.7.3 python-decouple==3.3 python-dotenv==0.14.0 python-editor==1.0.4 python-stdnum==1.9 pytz==2022.2.1 pyyaml==5.4 requests==2.27.1 requests-mock==1.5.2 requests-toolbelt==0.9.1 six==1.16.0 sortedcontainers==2.1.0 sqlalchemy==1.3.24 sqlalchemy-citext==1.3.post0 sqlalchemy-utils==0.33.11 tinycss2==1.1.1 tqdm==4.32.2 urllib3==1.26.12 weasyprint==44 webargs==5.5.3 webencodings==0.5.1 werkzeug==2.0.3 wtforms==3.0.1 xlrd==2.0.1 cryptography==39.0.1 Authlib==1.2.1 gunicorn==21.2.0
|
||||
|
||||
RUN pip install -i https://test.pypi.org/simple/ ereuseapitest==0.0.8
|
||||
|
||||
COPY . .
|
||||
RUN pip install -e .
|
||||
|
||||
COPY docker/devicehub.entrypoint.sh .
|
||||
ENTRYPOINT sh ./devicehub.entrypoint.sh
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
.git
|
||||
.env
|
||||
# TODO need to comment it to copy the entrypoint
|
||||
#docker
|
||||
Makefile
|
||||
|
||||
# Emacs backup files
|
||||
*~
|
||||
.\#*
|
||||
# Vim swap files
|
||||
*.swp
|
||||
*.swo
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
set -u
|
||||
# DEBUG
|
||||
set -x
|
||||
|
||||
# 3. Generate an environment .env file.
|
||||
gen_env_vars() {
|
||||
# generate config using env vars from docker
|
||||
cat > .env <<END
|
||||
DB_USER=${DB_USER}
|
||||
DB_PASSWORD=${DB_PASSWORD}
|
||||
DB_HOST=${DB_HOST}
|
||||
DB_DATABASE=${DB_DATABASE}
|
||||
SCHEMA=dbtest
|
||||
DB_SCHEMA=dbtest
|
||||
HOST=${HOST}
|
||||
EMAIL_DEMO=${EMAIL_DEMO}
|
||||
PASSWORD_DEMO=${PASSWORD_DEMO}
|
||||
JWT_PASS=${JWT_PASS}
|
||||
SECRET_KEY=${SECRET_KEY}
|
||||
|
||||
API_DLT=${API_DLT}
|
||||
API_RESOLVER=${API_RESOLVER}
|
||||
API_DLT_TOKEN=${API_DLT_TOKEN}
|
||||
ID_FEDERATED=${ID_FEDERATED}
|
||||
|
||||
OAUTH2_JWT_ENABLED=True
|
||||
OAUTH2_JWT_ISS=https://authlib.org
|
||||
OAUTH2_JWT_KEY=secret-key
|
||||
OAUTH2_JWT_ALG=HS256
|
||||
|
||||
URL_MANUALS=${URL_MANUALS}
|
||||
END
|
||||
}
|
||||
|
||||
wait_for_postgres() {
|
||||
# old one was
|
||||
#sleep 4
|
||||
|
||||
default_postgres_port=5432
|
||||
# thanks https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
|
||||
while ! nc -z ${DB_HOST} ${default_postgres_port}; do
|
||||
sleep 0.5
|
||||
done
|
||||
}
|
||||
|
||||
init_data() {
|
||||
|
||||
# 7. Run alembic of the project.
|
||||
alembic -x inventory=dbtest upgrade head
|
||||
# 8. Running alembic from oidc module.y
|
||||
cd ereuse_devicehub/modules/oidc
|
||||
alembic -x inventory=dbtest upgrade head
|
||||
cd -
|
||||
# 9. Running alembic from dpp module.
|
||||
cd ereuse_devicehub/modules/dpp/
|
||||
alembic -x inventory=dbtest upgrade head
|
||||
cd -
|
||||
|
||||
# 11. Generate a minimal data structure.
|
||||
# TODO it has some errors (?)
|
||||
flask initdata || true
|
||||
}
|
||||
|
||||
big_error() {
|
||||
local message="${@}"
|
||||
echo "###############################################" >&2
|
||||
echo "# ERROR: ${message}" >&2
|
||||
echo "###############################################" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
handle_federated_id() {
|
||||
|
||||
# devicehub host and id federated checker
|
||||
|
||||
EXPECTED_ID_FEDERATED="$(curl -s "${API_RESOLVER}/getAll" \
|
||||
| jq -r '.url | to_entries | .[] | select(.value == "'"${DEVICEHUB_HOST}"'") | .key' \
|
||||
| head -n 1)"
|
||||
|
||||
# if is a new DEVICEHUB_HOST, then register it
|
||||
if [ -z "${EXPECTED_ID_FEDERATED}" ]; then
|
||||
# TODO better docker compose run command
|
||||
cmd="docker compose run --entrypoint= devicehub flask dlt_insert_members ${DEVICEHUB_HOST}"
|
||||
big_error "No FEDERATED ID maybe you should run \`${cmd}\`"
|
||||
fi
|
||||
|
||||
# if not new DEVICEHUB_HOST, then check consistency
|
||||
|
||||
# if there is already an ID in the DLT, it should match with my internal ID
|
||||
if [ ! "${EXPECTED_ID_FEDERATED}" = "${ID_FEDERATED}" ]; then
|
||||
|
||||
big_error "ID_FEDERATED should be ${EXPECTED_ID_FEDERATED} instead of ${ID_FEDERATED}"
|
||||
fi
|
||||
|
||||
# not needed, but reserved
|
||||
# EXPECTED_DEVICEHUB_HOST="$(curl -s "${API_RESOLVER}/getAll" \
|
||||
# | jq -r '.url | to_entries | .[] | select(.key == "'"${ID_FEDERATED}"'") | .value' \
|
||||
# | head -n 1)"
|
||||
# if [ ! "${EXPECTED_DEVICEHUB_HOST}" = "${DEVICEHUB_HOST}" ]; then
|
||||
# big_error "ERROR: DEVICEHUB_HOST should be ${EXPECTED_DEVICEHUB_HOST} instead of ${DEVICEHUB_HOST}"
|
||||
# fi
|
||||
|
||||
}
|
||||
|
||||
main() {
|
||||
|
||||
gen_env_vars
|
||||
|
||||
wait_for_postgres
|
||||
|
||||
init_flagfile='/container_initialized'
|
||||
if [ ! -f "${init_flagfile}" ]; then
|
||||
|
||||
# 7, 8, 9, 11
|
||||
init_data
|
||||
|
||||
# 12. Add a new server to the 'api resolver'
|
||||
handle_federated_id
|
||||
|
||||
# 13. Do a rsync api resolve
|
||||
flask dlt_rsync_members
|
||||
|
||||
# 14. Register a new user to the DLT
|
||||
flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
|
||||
|
||||
# non DL user (only for the inventory)
|
||||
# flask adduser user2@dhub.com ${PASSWORD_DEMO}
|
||||
|
||||
# # 15. Add inventory snapshots for user "${EMAIL_DEMO}".
|
||||
cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
|
||||
/usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
|
||||
|
||||
# # 16.
|
||||
flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
|
||||
|
||||
# remain next command as the last operation for this if conditional
|
||||
touch "${init_flagfile}"
|
||||
fi
|
||||
|
||||
# 17. Use gunicorn
|
||||
# thanks https://akira3030.github.io/formacion/articulos/python-flask-gunicorn-docker.html
|
||||
# TODO meanwhile no nginx (step 19), gunicorn cannot serve static files, then we prefer development server
|
||||
#gunicorn --access-logfile - --error-logfile - --workers 4 -b :5000 app:app
|
||||
# alternative: run development server
|
||||
flask run --host=0.0.0.0 --port 5000
|
||||
|
||||
# DEBUG
|
||||
#sleep infinity
|
||||
}
|
||||
|
||||
main "${@}"
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
FROM postgres:15.4-bookworm
|
||||
# this is the latest in 2023-09-14_13-01-38
|
||||
#FROM postgres:latest
|
||||
|
||||
# Add a SQL script that will be executed upon container startup
|
||||
COPY docker/postgres.setupdb.sql /docker-entrypoint-initdb.d/
|
||||
|
||||
EXPOSE 5432
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
-- 6. Create the necessary extensions.
|
||||
CREATE EXTENSION pgcrypto SCHEMA public;
|
||||
CREATE EXTENSION ltree SCHEMA public;
|
||||
CREATE EXTENSION citext SCHEMA public;
|
||||
CREATE EXTENSION pg_trgm SCHEMA public;
|
||||
|
|
@ -5,6 +5,24 @@ DB_DATABASE='devicehub'
|
|||
API_DLT='http://$IP_API_DLT'
|
||||
API_DLT_TOKEN=$TOKEN
|
||||
API_RESOLVER='http://$IP_API_RESOLVER'
|
||||
ID_FEDERATED='DH12'
|
||||
# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
|
||||
ID_FEDERATED='$ID'
|
||||
URL_MANUALS='http://$IP_MANUALS'
|
||||
|
||||
DEVICEHUB_HOST='http://devicehub.example.com'
|
||||
HOST='localhost'
|
||||
|
||||
SCHEMA='dbtest'
|
||||
DB_SCHEMA='dbtest'
|
||||
|
||||
EMAIL_DEMO='user@example.org'
|
||||
PASSWORD_DEMO='changeme'
|
||||
|
||||
JWT_PASS='changeme'
|
||||
SECRET_KEY='changeme'
|
||||
|
||||
# important to import snapshots (step 15)
|
||||
# rel path starts with ./
|
||||
#SNAPSHOTS_PATH='./relpath/to/snapshots'
|
||||
# full path starts with /
|
||||
SNAPSHOTS_PATH='/fullpath/to/snapshots'
|
||||
|
|
|
|||
Reference in New Issue