From 907bf2dba09c604641a94ebf04f6b5dcb7f11c93 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 19 Sep 2023 18:03:23 +0200 Subject: [PATCH 01/17] add basic dockerization to devicehub dpp --- .gitignore | 3 + Makefile | 30 +++++ docker-compose.yml | 47 +++++++ docker/devicehub.Dockerfile | 30 +++++ docker/devicehub.Dockerfile.dockerignore | 12 ++ docker/devicehub.entrypoint.sh | 154 +++++++++++++++++++++++ docker/postgres.Dockerfile | 8 ++ docker/postgres.setupdb.sql | 5 + examples/env.example | 20 ++- 9 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100644 docker-compose.yml create mode 100644 docker/devicehub.Dockerfile create mode 100644 docker/devicehub.Dockerfile.dockerignore create mode 100755 docker/devicehub.entrypoint.sh create mode 100644 docker/postgres.Dockerfile create mode 100644 docker/postgres.setupdb.sql diff --git a/.gitignore b/.gitignore index e7c3d207..39dff2b0 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,6 @@ examples/create-db2.sh package-lock.json snapshots/ modules/ + +# emacs +*~ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..abc81f99 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +project := dkr-dsg.ac.upc.edu/devicehub + +branch := `git branch --show-current` +commit := `git log -1 --format=%h` +tag := ${branch}__${commit} + +# docker images +devicehub_image := ${project}/devicehub:${tag} +postgres_image := ${project}/postgres:${tag} + +# 2. Create a virtual environment. +docker_build: + docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . + # DEBUG + #docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . --progress=plain --no-cache + + docker build -f docker/postgres.Dockerfile -t ${postgres_image} . + # DEBUG + #docker build -f docker/postgres.Dockerfile -t ${postgres_image} . --progress=plain --no-cache + +docker_publish: + docker push ${devicehub_image} + +.PHONY: docker +docker: + $(MAKE) docker_build + #$(MAKE) docker_publish + @printf "\nimage: ${devicehub_image}\n" + @printf "\nimage: ${postgres_image}\n" + @printf "\ndocker images built and published\n" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..5224073b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +version: "3.9" +services: + + devicehub: + init: true + # TODO + image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp__0fb4fa5b + #build . + environment: + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=postgres + - DB_DATABASE=${DB_DATABASE} + - HOST=${HOST} + - EMAIL_DEMO=${EMAIL_DEMO} + - PASSWORD_DEMO=${PASSWORD_DEMO} + - JWT_PASS=${JWT_PASS} + - SECRET_KEY=${SECRET_KEY} + - API_DLT=${API_DLT} + - API_RESOLVER=${API_RESOLVER} + - API_DLT_TOKEN=${API_DLT_TOKEN} + - DEVICEHUB_HOST=${DEVICEHUB_HOST} + - ID_FEDERATED=${ID_FEDERATED} + - URL_MANUALS=${URL_MANUALS} + ports: + - 5000:5000 + volumes: + - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro + + postgres: + image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp__0fb4fa5b + # 4. To create the database. + # 5. Give permissions to the corresponding users in the database. + # extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables + environment: + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_USER=${DB_USER} + - POSTGRES_DB=${DB_DATABASE} + ports: + - 5432:5432 + # TODO persistence + #volumes: + # - pg_data:/var/lib/postgresql/data + + + # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/ + #nginx diff --git a/docker/devicehub.Dockerfile b/docker/devicehub.Dockerfile new file mode 100644 index 00000000..f14fc8a5 --- /dev/null +++ b/docker/devicehub.Dockerfile @@ -0,0 +1,30 @@ +FROM debian:bullseye-slim + +RUN apt update && apt-get install --no-install-recommends -y \ + python3-minimal \ + python3-pip \ + python-is-python3 \ + python3-psycopg2 \ + python3-dev \ + libpq-dev \ + build-essential \ + libpangocairo-1.0-0 \ + curl \ + jq \ + time \ + netcat + +WORKDIR /opt/devicehub + +# this is exactly the same as examples/pip_install.sh except the last command +# to improve the docker layer builds, it has been separated +RUN pip install --upgrade pip +RUN pip install alembic==1.8.1 anytree==2.8.0 apispec==0.39.0 atomicwrites==1.4.0 blinker==1.5 boltons==23.0.0 cairocffi==1.4.0 cairosvg==2.5.2 certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.0.12 click==6.7 click-spinner==0.1.8 colorama==0.3.9 colour==0.1.5 cssselect2==0.7.0 defusedxml==0.7.1 et-xmlfile==1.1.0 flask==1.0.2 flask-cors==3.0.10 flask-login==0.5.0 flask-sqlalchemy==2.5.1 flask-weasyprint==0.4 flask-wtf==1.0.0 hashids==1.2.0 html5lib==1.1 idna==3.4 inflection==0.5.1 itsdangerous==2.0.1 jinja2==3.0.3 mako==1.2.3 markupsafe==2.1.1 marshmallow==3.0.0b11 marshmallow-enum==1.4.1 more-itertools==8.12.0 numpy==1.22.0 odfpy==1.4.1 openpyxl==3.0.10 pandas==1.3.5 passlib==1.7.1 phonenumbers==8.9.11 pillow==9.2.0 pint==0.9 psycopg2-binary==2.8.3 py-dmidecode==0.1.0 pycparser==2.21 pyjwt==2.4.0 pyphen==0.13.0 python-dateutil==2.7.3 python-decouple==3.3 python-dotenv==0.14.0 python-editor==1.0.4 python-stdnum==1.9 pytz==2022.2.1 pyyaml==5.4 requests==2.27.1 requests-mock==1.5.2 requests-toolbelt==0.9.1 six==1.16.0 sortedcontainers==2.1.0 sqlalchemy==1.3.24 sqlalchemy-citext==1.3.post0 sqlalchemy-utils==0.33.11 tinycss2==1.1.1 tqdm==4.32.2 urllib3==1.26.12 weasyprint==44 webargs==5.5.3 webencodings==0.5.1 werkzeug==2.0.3 wtforms==3.0.1 xlrd==2.0.1 cryptography==39.0.1 Authlib==1.2.1 gunicorn==21.2.0 + +RUN pip install -i https://test.pypi.org/simple/ ereuseapitest==0.0.8 + +COPY . . +RUN pip install -e . + +COPY docker/devicehub.entrypoint.sh . +ENTRYPOINT sh ./devicehub.entrypoint.sh diff --git a/docker/devicehub.Dockerfile.dockerignore b/docker/devicehub.Dockerfile.dockerignore new file mode 100644 index 00000000..57848430 --- /dev/null +++ b/docker/devicehub.Dockerfile.dockerignore @@ -0,0 +1,12 @@ +.git +.env +# TODO need to comment it to copy the entrypoint +#docker +Makefile + +# Emacs backup files +*~ +.\#* +# Vim swap files +*.swp +*.swo diff --git a/docker/devicehub.entrypoint.sh b/docker/devicehub.entrypoint.sh new file mode 100755 index 00000000..a7d25dbb --- /dev/null +++ b/docker/devicehub.entrypoint.sh @@ -0,0 +1,154 @@ +#!/bin/sh + +set -e +set -u +# DEBUG +set -x + +# 3. Generate an environment .env file. +gen_env_vars() { + # generate config using env vars from docker + cat > .env <&2 + echo "# ERROR: ${message}" >&2 + echo "###############################################" >&2 + exit 1 +} + +handle_federated_id() { + + # devicehub host and id federated checker + + EXPECTED_ID_FEDERATED="$(curl -s "${API_RESOLVER}/getAll" \ + | jq -r '.url | to_entries | .[] | select(.value == "'"${DEVICEHUB_HOST}"'") | .key' \ + | head -n 1)" + + # if is a new DEVICEHUB_HOST, then register it + if [ -z "${EXPECTED_ID_FEDERATED}" ]; then + # TODO better docker compose run command + cmd="docker compose run --entrypoint= devicehub flask dlt_insert_members ${DEVICEHUB_HOST}" + big_error "No FEDERATED ID maybe you should run \`${cmd}\`" + fi + + # if not new DEVICEHUB_HOST, then check consistency + + # if there is already an ID in the DLT, it should match with my internal ID + if [ ! "${EXPECTED_ID_FEDERATED}" = "${ID_FEDERATED}" ]; then + + big_error "ID_FEDERATED should be ${EXPECTED_ID_FEDERATED} instead of ${ID_FEDERATED}" + fi + + # not needed, but reserved + # EXPECTED_DEVICEHUB_HOST="$(curl -s "${API_RESOLVER}/getAll" \ + # | jq -r '.url | to_entries | .[] | select(.key == "'"${ID_FEDERATED}"'") | .value' \ + # | head -n 1)" + # if [ ! "${EXPECTED_DEVICEHUB_HOST}" = "${DEVICEHUB_HOST}" ]; then + # big_error "ERROR: DEVICEHUB_HOST should be ${EXPECTED_DEVICEHUB_HOST} instead of ${DEVICEHUB_HOST}" + # fi + +} + +main() { + + gen_env_vars + + wait_for_postgres + + init_flagfile='/container_initialized' + if [ ! -f "${init_flagfile}" ]; then + + # 7, 8, 9, 11 + init_data + + # 12. Add a new server to the 'api resolver' + handle_federated_id + + # 13. Do a rsync api resolve + flask dlt_rsync_members + + # 14. Register a new user to the DLT + flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator + + # non DL user (only for the inventory) + # flask adduser user2@dhub.com ${PASSWORD_DEMO} + + # # 15. Add inventory snapshots for user "${EMAIL_DEMO}". + cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files + /usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO} + + # # 16. + flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO} + + # remain next command as the last operation for this if conditional + touch "${init_flagfile}" + fi + + # 17. Use gunicorn + # thanks https://akira3030.github.io/formacion/articulos/python-flask-gunicorn-docker.html + # TODO meanwhile no nginx (step 19), gunicorn cannot serve static files, then we prefer development server + #gunicorn --access-logfile - --error-logfile - --workers 4 -b :5000 app:app + # alternative: run development server + flask run --host=0.0.0.0 --port 5000 + + # DEBUG + #sleep infinity +} + +main "${@}" diff --git a/docker/postgres.Dockerfile b/docker/postgres.Dockerfile new file mode 100644 index 00000000..7ed15cf8 --- /dev/null +++ b/docker/postgres.Dockerfile @@ -0,0 +1,8 @@ +FROM postgres:15.4-bookworm +# this is the latest in 2023-09-14_13-01-38 +#FROM postgres:latest + +# Add a SQL script that will be executed upon container startup +COPY docker/postgres.setupdb.sql /docker-entrypoint-initdb.d/ + +EXPOSE 5432 diff --git a/docker/postgres.setupdb.sql b/docker/postgres.setupdb.sql new file mode 100644 index 00000000..a6f104a3 --- /dev/null +++ b/docker/postgres.setupdb.sql @@ -0,0 +1,5 @@ +-- 6. Create the necessary extensions. +CREATE EXTENSION pgcrypto SCHEMA public; +CREATE EXTENSION ltree SCHEMA public; +CREATE EXTENSION citext SCHEMA public; +CREATE EXTENSION pg_trgm SCHEMA public; diff --git a/examples/env.example b/examples/env.example index bfa97453..68366898 100644 --- a/examples/env.example +++ b/examples/env.example @@ -5,6 +5,24 @@ DB_DATABASE='devicehub' API_DLT='http://$IP_API_DLT' API_DLT_TOKEN=$TOKEN API_RESOLVER='http://$IP_API_RESOLVER' -ID_FEDERATED='DH12' +# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED +ID_FEDERATED='$ID' URL_MANUALS='http://$IP_MANUALS' +DEVICEHUB_HOST='http://devicehub.example.com' +HOST='localhost' + +SCHEMA='dbtest' +DB_SCHEMA='dbtest' + +EMAIL_DEMO='user@example.org' +PASSWORD_DEMO='changeme' + +JWT_PASS='changeme' +SECRET_KEY='changeme' + +# important to import snapshots (step 15) +# rel path starts with ./ +#SNAPSHOTS_PATH='./relpath/to/snapshots' +# full path starts with / +SNAPSHOTS_PATH='/fullpath/to/snapshots' From f37800dcd324b26d5707c415f60443a1bffc6c06 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 19 Sep 2023 18:15:16 +0200 Subject: [PATCH 02/17] docker: publish new image and put it in d-compose --- Makefile | 3 ++- docker-compose.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index abc81f99..0d4272f8 100644 --- a/Makefile +++ b/Makefile @@ -20,11 +20,12 @@ docker_build: docker_publish: docker push ${devicehub_image} + docker push ${postgres_image} .PHONY: docker docker: $(MAKE) docker_build - #$(MAKE) docker_publish + $(MAKE) docker_publish @printf "\nimage: ${devicehub_image}\n" @printf "\nimage: ${postgres_image}\n" @printf "\ndocker images built and published\n" diff --git a/docker-compose.yml b/docker-compose.yml index 5224073b..bd6c6f6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: devicehub: init: true # TODO - image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp__0fb4fa5b + image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__eafcac09 #build . environment: - DB_USER=${DB_USER} @@ -28,7 +28,7 @@ services: - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro postgres: - image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp__0fb4fa5b + image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp_docker__eafcac09 # 4. To create the database. # 5. Give permissions to the corresponding users in the database. # extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables From 260ac90f860154f6c8df8e401387f5dff488adb6 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 20 Sep 2023 00:11:58 +0200 Subject: [PATCH 03/17] reorder env vars in entrypoint for coherence --- docker/devicehub.entrypoint.sh | 39 ++++++++++++++++------------------ examples/env.example | 2 +- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/docker/devicehub.entrypoint.sh b/docker/devicehub.entrypoint.sh index a7d25dbb..e8c89a6b 100755 --- a/docker/devicehub.entrypoint.sh +++ b/docker/devicehub.entrypoint.sh @@ -9,29 +9,26 @@ set -x gen_env_vars() { # generate config using env vars from docker cat > .env < Date: Thu, 21 Sep 2023 18:15:54 +0200 Subject: [PATCH 04/17] automate OIDC setup for devicehub server & client --- docker-compose.yml | 70 ++++++++++++++++++++++++++++------ docker/devicehub.entrypoint.sh | 55 +++++++++++++++++++++++--- examples/env.example | 5 ++- 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bd6c6f6d..0454d948 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,13 @@ version: "3.9" services: - devicehub: + devicehub-id-server: init: true - # TODO - image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__eafcac09 - #build . + image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__54511e1b environment: - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} - - DB_HOST=postgres + - DB_HOST=postgres-id-server - DB_DATABASE=${DB_DATABASE} - HOST=${HOST} - EMAIL_DEMO=${EMAIL_DEMO} @@ -19,15 +17,18 @@ services: - API_DLT=${API_DLT} - API_RESOLVER=${API_RESOLVER} - API_DLT_TOKEN=${API_DLT_TOKEN} - - DEVICEHUB_HOST=${DEVICEHUB_HOST} - - ID_FEDERATED=${ID_FEDERATED} + - DEVICEHUB_HOST=${SERVER_ID_DEVICEHUB_HOST} + - ID_FEDERATED=${SERVER_ID_FEDERATED} - URL_MANUALS=${URL_MANUALS} + - ID_SERVICE=${SERVER_ID_SERVICE} + - AUTHORIZED_CLIENT_URL=${CLIENT_ID_DEVICEHUB_HOST} ports: - 5000:5000 volumes: - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro + - shared:/shared:rw - postgres: + postgres-id-server: image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp_docker__eafcac09 # 4. To create the database. # 5. Give permissions to the corresponding users in the database. @@ -36,12 +37,59 @@ services: - POSTGRES_PASSWORD=${DB_PASSWORD} - POSTGRES_USER=${DB_USER} - POSTGRES_DB=${DB_DATABASE} + # DEBUG + #ports: + # - 5432:5432 + # TODO persistence + #volumes: + # - pg_data:/var/lib/postgresql/data + + devicehub-id-client: + init: true + image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__54511e1b + environment: + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=postgres-id-client + - DB_DATABASE=${DB_DATABASE} + - HOST=${HOST} + - EMAIL_DEMO=${EMAIL_DEMO} + - PASSWORD_DEMO=${PASSWORD_DEMO} + - JWT_PASS=${JWT_PASS} + - SECRET_KEY=${SECRET_KEY} + - API_DLT=${API_DLT} + - API_RESOLVER=${API_RESOLVER} + - API_DLT_TOKEN=${API_DLT_TOKEN} + - DEVICEHUB_HOST=${CLIENT_ID_DEVICEHUB_HOST} + - SERVER_ID_HOST=${SERVER_ID_DEVICEHUB_HOST} + - ID_FEDERATED=${CLIENT_ID_FEDERATED} + - URL_MANUALS=${URL_MANUALS} + - ID_SERVICE=${CLIENT_ID_SERVICE} ports: - - 5432:5432 + - 5001:5001 + volumes: + - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro + - shared:/shared:ro + + postgres-id-client: + image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp_docker__eafcac09 + # 4. To create the database. + # 5. Give permissions to the corresponding users in the database. + # extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables + environment: + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_USER=${DB_USER} + - POSTGRES_DB=${DB_DATABASE} + # DEBUG + #ports: + # - 5432:5432 # TODO persistence #volumes: # - pg_data:/var/lib/postgresql/data - # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/ - #nginx + # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/ + #nginx + +volumes: + shared: diff --git a/docker/devicehub.entrypoint.sh b/docker/devicehub.entrypoint.sh index e8c89a6b..9783b367 100755 --- a/docker/devicehub.entrypoint.sh +++ b/docker/devicehub.entrypoint.sh @@ -102,15 +102,48 @@ handle_federated_id() { } -main() { +config_oidc() { + # TODO test allowing more than 1 client + if [ "${ID_SERVICE}" = "server_id" ]; then - gen_env_vars + client_description="client identity from docker compose demo" - wait_for_postgres + # in AUTHORIZED_CLIENT_URL we remove anything before :// + flask add_contract_oidc \ + "${EMAIL_DEMO}" \ + "${client_description}" \ + "${AUTHORIZED_CLIENT_URL}" \ + > /shared/client_id_${AUTHORIZED_CLIENT_URL#*://} - init_flagfile='/container_initialized' + elif [ "${ID_SERVICE}" = "client_id" ]; then + + # in DEVICEHUB_HOST we remove anything before :// + CLIENT_ID_CONFIG="/shared/client_id_${DEVICEHUB_HOST#*://}" + + # wait that the file generated by the server_id is readable + while true; do + if [ -f "${CLIENT_ID_CONFIG}" ]; then + break + fi + sleep 1 + done + + client_id="$(cat "${CLIENT_ID_CONFIG}" | jq -r '.client_id')" + client_secret="$(cat "${CLIENT_ID_CONFIG}" | jq -r '.client_secret')" + + flask add_client_oidc \ + "${SERVER_ID_HOST}" \ + "${client_id}" \ + "${client_secret}" + + else + big_error "Something went wrong ${ID_SERVICE} is not server_id nor client_id" + fi +} + +config_phase() { + init_flagfile='/already_configured' if [ ! -f "${init_flagfile}" ]; then - # 7, 8, 9, 11 init_data @@ -133,9 +166,21 @@ main() { # # 16. flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO} + # config server or client ID + config_oidc + # remain next command as the last operation for this if conditional touch "${init_flagfile}" fi +} + +main() { + + gen_env_vars + + wait_for_postgres + + config_phase # 17. Use gunicorn # thanks https://akira3030.github.io/formacion/articulos/python-flask-gunicorn-docker.html diff --git a/examples/env.example b/examples/env.example index 8b6d62d2..476f5941 100644 --- a/examples/env.example +++ b/examples/env.example @@ -9,7 +9,10 @@ API_RESOLVER='http://$IP_API_RESOLVER' ID_FEDERATED='$ID' URL_MANUALS='http://$IP_MANUALS' -DEVICEHUB_HOST='http://devicehub.example.com' +SERVER_ID_DEVICEHUB_HOST='http://devicehub-server-id.example.com' +CLIENT_ID_DEVICEHUB_HOST='http://devicehub-client-id.example.com' +SERVER_ID_SERVICE='server_id' +CLIENT_ID_SERVICE='client_id' HOST='localhost' SCHEMA='dbtest' From 37069ff561e2bf50dd723acadfe3722058c6f60b Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 18:43:59 +0200 Subject: [PATCH 05/17] bugfix docker compose devicehub client id --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0454d948..a0660179 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,7 +66,7 @@ services: - URL_MANUALS=${URL_MANUALS} - ID_SERVICE=${CLIENT_ID_SERVICE} ports: - - 5001:5001 + - 5001:5000 volumes: - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro - shared:/shared:ro From 9dec42bd051d34488fecf15cfd933844991e2a91 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 19:07:14 +0200 Subject: [PATCH 06/17] docker-compose: update devicehub image --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a0660179..4c053323 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: devicehub-id-server: init: true - image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__54511e1b + image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__37069ff5 environment: - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} @@ -46,7 +46,7 @@ services: devicehub-id-client: init: true - image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__54511e1b + image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__37069ff5 environment: - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} From 7a85ebd8f8fc73ee3daa912b158d2e0d9544149f Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 20:57:00 +0200 Subject: [PATCH 07/17] client_id_config is not a global env var then, move it to downcase to avoid confusion --- docker/devicehub.entrypoint.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/devicehub.entrypoint.sh b/docker/devicehub.entrypoint.sh index 9783b367..800d04c7 100755 --- a/docker/devicehub.entrypoint.sh +++ b/docker/devicehub.entrypoint.sh @@ -118,18 +118,18 @@ config_oidc() { elif [ "${ID_SERVICE}" = "client_id" ]; then # in DEVICEHUB_HOST we remove anything before :// - CLIENT_ID_CONFIG="/shared/client_id_${DEVICEHUB_HOST#*://}" + client_id_config="/shared/client_id_${DEVICEHUB_HOST#*://}" # wait that the file generated by the server_id is readable while true; do - if [ -f "${CLIENT_ID_CONFIG}" ]; then + if [ -f "${client_id_config}" ]; then break fi sleep 1 done - client_id="$(cat "${CLIENT_ID_CONFIG}" | jq -r '.client_id')" - client_secret="$(cat "${CLIENT_ID_CONFIG}" | jq -r '.client_secret')" + client_id="$(cat "${client_id_config}" | jq -r '.client_id')" + client_secret="$(cat "${client_id_config}" | jq -r '.client_secret')" flask add_client_oidc \ "${SERVER_ID_HOST}" \ From 2c4b0006ccd1a702a933bfb634e2f0afd91c9809 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 21:33:11 +0200 Subject: [PATCH 08/17] bugfix oidc client not working the file gets created, but you need to wait some time to get data into it --- docker/devicehub.entrypoint.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/devicehub.entrypoint.sh b/docker/devicehub.entrypoint.sh index 800d04c7..220cdef6 100755 --- a/docker/devicehub.entrypoint.sh +++ b/docker/devicehub.entrypoint.sh @@ -119,18 +119,21 @@ config_oidc() { # in DEVICEHUB_HOST we remove anything before :// client_id_config="/shared/client_id_${DEVICEHUB_HOST#*://}" + client_id= + client_secret= # wait that the file generated by the server_id is readable while true; do if [ -f "${client_id_config}" ]; then - break + client_id="$(cat "${client_id_config}" | jq -r '.client_id')" + client_secret="$(cat "${client_id_config}" | jq -r '.client_secret')" + if [ "${client_id}" ] && [ "${client_secret}" ]; then + break + fi fi sleep 1 done - client_id="$(cat "${client_id_config}" | jq -r '.client_id')" - client_secret="$(cat "${client_id_config}" | jq -r '.client_secret')" - flask add_client_oidc \ "${SERVER_ID_HOST}" \ "${client_id}" \ From 6a58dcc68f4eec3958fbea8e2b7c4f1358ad3b92 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 21:37:37 +0200 Subject: [PATCH 09/17] refactor Makefile - use ereuse project (to avoid confusion between devicehub project and devicehub image) - facilitate the docker image URL on the make docker_build --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0d4272f8..a51c1cf5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -project := dkr-dsg.ac.upc.edu/devicehub +project := dkr-dsg.ac.upc.edu/ereuse branch := `git branch --show-current` commit := `git log -1 --format=%h` @@ -17,6 +17,11 @@ docker_build: docker build -f docker/postgres.Dockerfile -t ${postgres_image} . # DEBUG #docker build -f docker/postgres.Dockerfile -t ${postgres_image} . --progress=plain --no-cache + @printf "\n##########################\n" + @printf "\ndevicehub image: ${devicehub_image}\n" + @printf "postgres image: ${postgres_image}\n" + @printf "\ndocker images built\n" + @printf "\n##########################\n\n" docker_publish: docker push ${devicehub_image} @@ -26,6 +31,4 @@ docker_publish: docker: $(MAKE) docker_build $(MAKE) docker_publish - @printf "\nimage: ${devicehub_image}\n" - @printf "\nimage: ${postgres_image}\n" - @printf "\ndocker images built and published\n" + @printf "\ndocker images published\n" From 5a965e245eda9ceaf8de4da510cd9bfb24465ce1 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 21:40:16 +0200 Subject: [PATCH 10/17] docker-compose: use unique users for each instance --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4c053323..fc64971f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - DB_HOST=postgres-id-server - DB_DATABASE=${DB_DATABASE} - HOST=${HOST} - - EMAIL_DEMO=${EMAIL_DEMO} + - EMAIL_DEMO=user5000@dhub.com - PASSWORD_DEMO=${PASSWORD_DEMO} - JWT_PASS=${JWT_PASS} - SECRET_KEY=${SECRET_KEY} @@ -53,7 +53,7 @@ services: - DB_HOST=postgres-id-client - DB_DATABASE=${DB_DATABASE} - HOST=${HOST} - - EMAIL_DEMO=${EMAIL_DEMO} + - EMAIL_DEMO=user5001@dhub.com - PASSWORD_DEMO=${PASSWORD_DEMO} - JWT_PASS=${JWT_PASS} - SECRET_KEY=${SECRET_KEY} From 2dc40e95fe7111fa3d11a5d6a1012e17f2705df7 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 21:40:29 +0200 Subject: [PATCH 11/17] docker-compose: update images --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fc64971f..e246a4c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: devicehub-id-server: init: true - image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__37069ff5 + image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp_docker__2c4b0006 environment: - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} @@ -29,7 +29,7 @@ services: - shared:/shared:rw postgres-id-server: - image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp_docker__eafcac09 + image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp_docker__2c4b0006 # 4. To create the database. # 5. Give permissions to the corresponding users in the database. # extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables @@ -46,7 +46,7 @@ services: devicehub-id-client: init: true - image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp_docker__37069ff5 + image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp_docker__2c4b0006 environment: - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} @@ -72,7 +72,7 @@ services: - shared:/shared:ro postgres-id-client: - image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp_docker__eafcac09 + image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp_docker__2c4b0006 # 4. To create the database. # 5. Give permissions to the corresponding users in the database. # extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables From dce28731588dbc3d1db74da4c0f3171f0c5a6fc8 Mon Sep 17 00:00:00 2001 From: pedro Date: Thu, 21 Sep 2023 22:12:03 +0200 Subject: [PATCH 12/17] update env.example: we need real reachable URLs specially for the OIDC demo --- examples/env.example | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/env.example b/examples/env.example index 476f5941..a97f9373 100644 --- a/examples/env.example +++ b/examples/env.example @@ -9,8 +9,10 @@ API_RESOLVER='http://$IP_API_RESOLVER' ID_FEDERATED='$ID' URL_MANUALS='http://$IP_MANUALS' -SERVER_ID_DEVICEHUB_HOST='http://devicehub-server-id.example.com' -CLIENT_ID_DEVICEHUB_HOST='http://devicehub-client-id.example.com' +#SERVER_ID_DEVICEHUB_HOST='http://devicehub-server-id.example.com' +SERVER_ID_DEVICEHUB_HOST='http://localhost:5000' +#CLIENT_ID_DEVICEHUB_HOST='http://devicehub-client-id.example.com' +CLIENT_ID_DEVICEHUB_HOST='http://localhost:5001' SERVER_ID_SERVICE='server_id' CLIENT_ID_SERVICE='client_id' HOST='localhost' From 843324bd174c7479ffe8df6f4e67c79391aa2f9c Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Sep 2023 08:17:47 +0200 Subject: [PATCH 13/17] fix strip slash in domain --- README.md => README_MANUAL_INSTALLATION.md | 0 ereuse_devicehub/config.py | 3 +++ ereuse_devicehub/modules/oidc/commands/insert_member_in_dlt.py | 3 +++ ereuse_devicehub/modules/oidc/commands/sync_dlt.py | 2 ++ 4 files changed, 8 insertions(+) rename README.md => README_MANUAL_INSTALLATION.md (100%) diff --git a/README.md b/README_MANUAL_INSTALLATION.md similarity index 100% rename from README.md rename to README_MANUAL_INSTALLATION.md diff --git a/ereuse_devicehub/config.py b/ereuse_devicehub/config.py index f93c1b4b..5152bee5 100644 --- a/ereuse_devicehub/config.py +++ b/ereuse_devicehub/config.py @@ -105,3 +105,6 @@ class DevicehubConfig(Config): OAUTH2_JWT_ISS = config('OAUTH2_JWT_ISS', '') OAUTH2_JWT_KEY = config('OAUTH2_JWT_KEY', None) OAUTH2_JWT_ALG = config('OAUTH2_JWT_ALG', 'HS256') + + if API_DLT: + API_DLT = API_DLT.strip("/") diff --git a/ereuse_devicehub/modules/oidc/commands/insert_member_in_dlt.py b/ereuse_devicehub/modules/oidc/commands/insert_member_in_dlt.py index e1c464bb..c944625f 100644 --- a/ereuse_devicehub/modules/oidc/commands/insert_member_in_dlt.py +++ b/ereuse_devicehub/modules/oidc/commands/insert_member_in_dlt.py @@ -21,6 +21,9 @@ class InsertMember: print("Error: you need a entry var API_RESOLVER in .env") return + api = api.strip("/") + domain = domain.strip("/") + data = {"url": domain} url = api + '/registerURL' res = requests.post(url, json=data) diff --git a/ereuse_devicehub/modules/oidc/commands/sync_dlt.py b/ereuse_devicehub/modules/oidc/commands/sync_dlt.py index 585a1803..5c75cb71 100644 --- a/ereuse_devicehub/modules/oidc/commands/sync_dlt.py +++ b/ereuse_devicehub/modules/oidc/commands/sync_dlt.py @@ -19,6 +19,8 @@ class GetMembers: print("Error: you need a entry var API_RESOLVER in .env") return + api = api.strip("/") + url = api + '/getAll' res = requests.get(url) if res.status_code != 200: From 82bf535915b32674a5ab2db3ee136f6924414afd Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Sep 2023 08:18:24 +0200 Subject: [PATCH 14/17] new readme --- README.md | 68 ++++++++++++++++++++++++++++++++++++++++ examples/env.example | 32 ++++++++++--------- examples/snapshot01.json | 1 + 3 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 README.md create mode 100644 examples/snapshot01.json diff --git a/README.md b/README.md new file mode 100644 index 00000000..2e44aa8a --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# Devicehub + +Devicehub is a distributed IT Asset Management System focused in reusing devices, created under the project [eReuse.org](https://www.ereuse.org) + +This README explains how to install and use Devicehub. [The documentation](http://devicehub.ereuse.org) explains the concepts and the API. + +Devicehub is built with [Teal](https://github.com/ereuse/teal) and [Flask](http://flask.pocoo.org). + +# Installing +Please visit the [Manual Installation](#README_MANUAL_INSTALLATION.md) for understand how you can install locally or deploy in a server. + +# Docker +You have a docker compose file for to do a automated deployment. In the next steps we can see as run and use. + +1. Download the sources: +``` + git clone https://github.com/eReuse/devicehub-teal.git + cd devicehub-teal +``` + +2. You need decide one dir in your system for share documents between your system and the dockers. +For us only as example we use "/tmp/dhub/" and need create it: +``` + mkdir /tmp/dhub +``` + +3. Copy your snapshots in this directory. If you don't have any snapshots copy one of the example directory. +``` + cp examples/snapshot01.json /tmp/dhub +``` + +4. Modify the file with environment variables in the file .env You can see one example in examples/env +If you don't have one please copy the examples/env file and modify the basic vars +``` + cp examples/env.example .env +``` +You can use this parameters for default as a test, but you need add values in this three variables: +``` + API_DLT + API_DLT_TOKEN + API_RESOLVER +``` + +5. run the dockers: +``` + docker compose up +``` +For stop the docker you can use Ctl+c and if you run again "compose up" you maintain the datas and infrastructure. + +6. If you want down the volumens and remove the datas, you can use: +``` + docker compose down -v +``` + +7. If you want to enter a shell inside a new container: +``` + docker run -it --entrypoint= ${target_docker_image} bash +``` + +If you want to enter a shell on already running container: +``` + docker exec -it ${target_docker_image} bash +``` + +For to know the valid value for ${target_docker_image} you can use: +``` + docker ps +` diff --git a/examples/env.example b/examples/env.example index a97f9373..eacd34a6 100644 --- a/examples/env.example +++ b/examples/env.example @@ -1,13 +1,20 @@ +# Please fill in these three variables +API_DLT='{ONE_URL}' +API_DLT_TOKEN='{ONE_TOKEN}' +API_RESOLVER='{ONE_URL}' + +# Database Variables DB_USER='dhub' DB_PASSWORD='ereuse' DB_HOST='localhost' -DB_DATABASE='devicehub' -API_DLT='http://$IP_API_DLT' -API_DLT_TOKEN='$TOKEN' -API_RESOLVER='http://$IP_API_RESOLVER' +DB_DATABASE='dpp' +SCHEMA='dbtest' +DB_SCHEMA='dbtest' + # TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED -ID_FEDERATED='$ID' -URL_MANUALS='http://$IP_MANUALS' +SERVER_ID_FEDERATED='DH12' +CLIENT_ID_FEDERATED='DH20' +URL_MANUALS='http://localhost:4000' #SERVER_ID_DEVICEHUB_HOST='http://devicehub-server-id.example.com' SERVER_ID_DEVICEHUB_HOST='http://localhost:5000' @@ -17,17 +24,14 @@ SERVER_ID_SERVICE='server_id' CLIENT_ID_SERVICE='client_id' HOST='localhost' -SCHEMA='dbtest' -DB_SCHEMA='dbtest' +EMAIL_DEMO='user@example.com' +PASSWORD_DEMO='1234' -EMAIL_DEMO='user@example.org' -PASSWORD_DEMO='changeme' - -JWT_PASS='changeme' -SECRET_KEY='changeme' +JWT_PASS='7KU4ZzsEfe' +SECRET_KEY='ffa96bac894ccaa070c23dfe18fd900eda2eebcbd5dac8b34db9b54ecec81501' # important to import snapshots (step 15) # rel path starts with ./ #SNAPSHOTS_PATH='./relpath/to/snapshots' # full path starts with / -SNAPSHOTS_PATH='/fullpath/to/snapshots' +SNAPSHOTS_PATH='/tmp/dhub_docker/snapshots' diff --git a/examples/snapshot01.json b/examples/snapshot01.json new file mode 100644 index 00000000..5697413f --- /dev/null +++ b/examples/snapshot01.json @@ -0,0 +1 @@ +{"closed": true, "components": [{"actions": [], "manufacturer": "Intel Corporation", "model": "82579LM Gigabit Network Connection", "serialNumber": "00:11:11:11:11:00", "speed": 1000.0, "type": "NetworkAdapter", "variant": "04", "wireless": false}, {"actions": [], "manufacturer": "Intel Corporation", "model": "7 Series/C216 Chipset Family High Definition Audio Controller", "serialNumber": null, "type": "SoundCard"}, {"actions": [], "format": "DIMM", "interface": "DDR3", "manufacturer": "Micron", "model": "16KTF51264AZ", "serialNumber": "AAAAAAAA", "size": 4096.0, "speed": 1600.0, "type": "RamModule"}, {"actions": [{"endTime": "2022-10-11T13:45:31.239555+00:00", "severity": "Info", "startTime": "2021-10-11T09:45:19.623967+00:00", "steps": [{"endTime": "2021-10-11T11:05:28.090897+00:00", "severity": "Info", "startTime": "2021-10-11T09:45:19.624163+00:00", "type": "StepZero"}, {"endTime": "2021-10-11T13:45:31.239402+00:00", "severity": "Info", "startTime": "2021-10-11T11:05:28.091255+00:00", "type": "StepRandom"}], "type": "EraseSectors"}, {"assessment": true, "commandTimeout": 30, "currentPendingSectorCount": 0, "elapsed": 60, "length": "Short", "lifetime": 18720, "offlineUncorrectable": 0, "powerCycleCount": 2147, "reallocatedSectorCount": 0, "reportedUncorrectableErrors": 0, "severity": "Info", "status": "Completed without error", "type": "TestDataStorage"}, {"elapsed": 11, "readSpeed": 119.0, "type": "BenchmarkDataStorage", "writeSpeed": 32.7}], "interface": "ATA", "manufacturer": "Seagate", "model": "ST3500418AS", "serialNumber": "AAAAAAAA", "size": 500000.0, "type": "HardDrive", "variant": "CC46"}, {"actions": [{"elapsed": 0, "rate": 25540.36, "type": "BenchmarkProcessor"}, {"elapsed": 8, "rate": 7.6939, "type": "BenchmarkProcessorSysbench"}], "address": 64, "brand": "Core i5", "cores": 4, "generation": 3, "manufacturer": "Intel Corp.", "model": "Intel Core i5-3470 CPU @ 3.20GHz", "serialNumber": null, "speed": 1.6242180000000002, "threads": 4, "type": "Processor"}, {"actions": [], "manufacturer": "Intel Corporation", "memory": null, "model": "Xeon E3-1200 v2/3rd Gen Core processor Graphics Controller", "serialNumber": null, "type": "GraphicCard"}, {"actions": [], "biosDate": "2012-08-07T00:00:00", "firewire": 0, "manufacturer": "LENOVO", "model": "MAHOBAY", "pcmcia": 0, "ramMaxSize": 32, "ramSlots": 4, "serial": 1, "serialNumber": null, "slots": 4, "type": "Motherboard", "usb": 3, "version": "9SKT39AUS"}], "device": {"actions": [{"elapsed": 1, "rate": 0.6507, "type": "BenchmarkRamSysbench"}], "chassis": "Tower", "manufacturer": "LENOVO", "model": "3227A2G", "serialNumber": "AAAAAAAA", "sku": "LENOVO_MT_3227", "type": "Desktop", "version": "ThinkCentre M92P"}, "elapsed": 187302510, "endTime": "2016-11-03T17:17:01.116554+00:00", "software": "Workbench", "type": "Snapshot", "uuid": "ae913de1-e639-476a-ad9b-78eabbe4628b", "version": "11.0b11"} \ No newline at end of file From 7e088eefc880e8d38296e919bf81555252283084 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Sep 2023 08:24:03 +0200 Subject: [PATCH 15/17] new readme --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e44aa8a..2d979674 100644 --- a/README.md +++ b/README.md @@ -65,4 +65,10 @@ If you want to enter a shell on already running container: For to know the valid value for ${target_docker_image} you can use: ``` docker ps -` +``` + +8. This are the details for use this implementation: + *devicehub with port 5000* is the identity provider of oidc and have user *user5000@example.com* + *devicehub with port 5001* is the client identity of oidc and have user *user5001@example.com* + + You can change this values in the file *.env* From b614fad41fd90bf579c0f24a25aeed2200ae6079 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Sep 2023 08:26:10 +0200 Subject: [PATCH 16/17] fix readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2d979674..a5935c81 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,9 @@ For to know the valid value for ${target_docker_image} you can use: ``` 8. This are the details for use this implementation: +``` *devicehub with port 5000* is the identity provider of oidc and have user *user5000@example.com* *devicehub with port 5001* is the client identity of oidc and have user *user5001@example.com* +``` You can change this values in the file *.env* From 68c342ee186a11f9ede05988899bb500247d60aa Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Sep 2023 08:27:05 +0200 Subject: [PATCH 17/17] fix readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a5935c81..5c56c5d5 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,9 @@ For to know the valid value for ${target_docker_image} you can use: ``` 8. This are the details for use this implementation: -``` + *devicehub with port 5000* is the identity provider of oidc and have user *user5000@example.com* + *devicehub with port 5001* is the client identity of oidc and have user *user5001@example.com* -``` You can change this values in the file *.env*