From 27742db4fbd6c8beca75e01c4cb52fbff1919970 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 10 Oct 2023 10:07:46 +0200 Subject: [PATCH] init with previous work this repo is a reinitialization with a change in the concept previous repo had git subtree of django-musician and django-orchestra, that was too static for our interest next approach is a script that pulls git repos, and that would be needed if you want to build everything, if not, just with the docker compose you would have enough for running the containers, hence, downloading the images from the docker registry --- .env.example | 6 ++ .gitignore | 4 + Makefile | 25 ++++++ README.md | 29 ++++++ docker-compose.yml | 37 ++++++++ docker/musician.Dockerfile | 26 ++++++ docker/musician.entrypoint.sh | 21 +++++ docker/nginx-orchestra-api.nginx.conf | 36 ++++++++ docker/orchestra.Dockerfile | 125 ++++++++++++++++++++++++++ docker/orchestra.entrypoint.sh | 30 +++++++ docker/orchestra.migrate.exp | 61 +++++++++++++ 11 files changed, 400 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 docker/musician.Dockerfile create mode 100755 docker/musician.entrypoint.sh create mode 100644 docker/nginx-orchestra-api.nginx.conf create mode 100644 docker/orchestra.Dockerfile create mode 100755 docker/orchestra.entrypoint.sh create mode 100644 docker/orchestra.migrate.exp diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1da8229 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +ORCHESTRA_SECRET_KEY= +MUSICIAN_SECRET_KEY= + +# specially useful if you want to deploy in a specific domain +#MUSICIAN_API_BASE_URL=https://orchestra.example.org +#ALLOWED_HOSTS=musician.example.org diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ca058a8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# protect env var secrets +.env +# emacs +*~ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3cbb905 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +project := dkr-dsg.ac.upc.edu/trustchain-oc1-orchestral + +branch := `git branch --show-current` +commit := `git log -1 --format=%h` +tag := ${branch}__${commit} + +# docker images +orchestra_image := ${project}/orchestra:${tag} +musician_image := ${project}/musician:${tag} + +docker_build: + docker build -f docker/orchestra.Dockerfile -t ${orchestra_image} . + docker build -f docker/musician.Dockerfile -t ${musician_image} . + +docker_publish: + docker push ${orchestra_image} + docker push ${musician_image} + +.PHONY: docker +docker: + $(MAKE) docker_build + $(MAKE) docker_publish + @printf "\nimage: ${orchestra_image}\n" + @printf "\nimage: ${musician_image}\n" + @printf "\ndocker images built and published\n" diff --git a/README.md b/README.md new file mode 100644 index 0000000..12c95fb --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +docker files and integrations + +# deploy everything in localhost + +note: right now the same applies for localhost and reachable deployments + +``` +docker compose up +``` + +# building and deploying new docker images + +``` +make docker +``` + +# dev + +if you want to enter a shell inside a new container: + +``` +docker run -it --entrypoint= ${target_docker_image} bash +``` + +if you want to enter a shell on already running container: + +``` +docker exec -it ${target_docker_image} bash +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2cc1f16 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: "3.9" +services: + + orchestra: + init: true + image: dkr-dsg.ac.upc.edu/trustchain-oc1-orchestral/orchestra:add_musician__95b0ed93 + environment: + - SECRET_KEY=${ORCHESTRA_SECRET_KEY:-123456} + ports: + - "9080:9080" + # TODO configure volumes + #volumes: + # - .:/home + + musician: + init: true + image: dkr-dsg.ac.upc.edu/trustchain-oc1-orchestral/musician:add_musician__95b0ed93 + ports: + - "8080:8080" + environment: + - SECRET_KEY=${MUSICIAN_SECRET_KEY:-123456} + - API_BASE_URL=${MUSICIAN_API_BASE_URL:-http://nginx-orchestra-api:3000} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-*} + # TODO configure volumes + #volumes: + # - .:/home + + # WARNING: this containers is hardcoded and is only useful in localhost deployments + # and as a reference for reachable deployments + nginx-orchestra-api: + image: nginx + ports: + - 3000:3000 + volumes: + # src https://hub.docker.com/_/nginx + # src https://github.com/docker-library/docs/tree/master/nginx#complex-configuration + - ./docker/nginx-orchestra-api.nginx.conf:/etc/nginx/nginx.conf:ro diff --git a/docker/musician.Dockerfile b/docker/musician.Dockerfile new file mode 100644 index 0000000..37a5765 --- /dev/null +++ b/docker/musician.Dockerfile @@ -0,0 +1,26 @@ +# right now this is this is heavily inspired to git repo django-musician/Dockerfile +#FROM python +FROM debian:bullseye-slim + +RUN apt update && apt-get install -y \ + python3-minimal \ + python3-pip \ + python3-dev \ + python-is-python3 + +WORKDIR /home + +RUN python3 -m pip install --upgrade pip +RUN pip install wheel + +COPY django-musician . +RUN pip install -r requirements.txt + +COPY docker/musician.entrypoint.sh . +ENTRYPOINT sh ./musician.entrypoint.sh + +#RUN python manage.py migrate +# +#EXPOSE 8080 +# +#ENTRYPOINT [ "python", "manage.py", "runserver", "0.0.0.0:8080" ] diff --git a/docker/musician.entrypoint.sh b/docker/musician.entrypoint.sh new file mode 100755 index 0000000..ab16c03 --- /dev/null +++ b/docker/musician.entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e +set -u +#set -x + +# go to the same path as the script +cd "$(dirname ${0})" + +cat > .env < python3.9 (dependency with celery should be removed), celery3.1 won't support django4.0 +# orchestra initially was using celery to run async tasks, but then switched to process/threads without MQ... but the dependency with celery was never fully removed :( +# django-iban wont support > 4.0 (django-iban is deprecated, replace by django-localflavor) +# django 3 cannot work https://stackoverflow.com/questions/59261254/no-module-named-django-contrib-staticfiles-templatetags +# RUN pip3 install \ +# django==3.2.18 \ +# django-fluent-dashboard \ +# django-admin-tools \ +# django-extensions \ +# celery==3.1.23 \ +# django-celery==3.3.1 \ +# django-cors-headers \ +# Markdown \ +# djangorestframework \ +# ecdsa \ +# Pygments \ +# django-filter \ +# jsonfield \ +# python-dateutil \ +# https://github.com/glic3rinu/passlib/archive/master.zip \ +# django-iban \ +# requests \ +# phonenumbers \ +# django-countries \ +# django-localflavor \ +# amqp \ +# pytz \ +# cracklib \ +# lxml + +# TODO maybe from here goes to docker entrypoint? +# TODO assumes that the project already exists, and in some cases that would be interesting +#COPY django-orchestra /home/orchestra +COPY django-orchestra /home/orchestra + +# this is to ensure django project is created on top of this working directory +WORKDIR /home/orchestra/ + +# TODO fix this better in the repo itself +RUN pip3 install -r requirements.txt +RUN pip3 install lxml==4.9.3 +#RUN sed -i 's/lxml==3.3.5/lxml==4.9.3/' requirements.txt +# solves "No module named 'orchestra'" +RUN pip install -e . + +RUN django-admin startproject panel --template=/home/orchestra/orchestra/conf/project_template/ + +RUN adduser orchestra \ + && sudo adduser orchestra sudo \ + && su - orchestra + +ENV PATH=$PATH:/home/orchestra/django-orchestra/orchestra/bin + +WORKDIR /home/orchestra/panel +COPY docker/orchestra.migrate.exp /home/orchestra/panel + +COPY docker/orchestra.entrypoint.sh . +ENTRYPOINT sh ./orchestra.entrypoint.sh + +# RUN expect -f ./orchestra.migrate.exp +# +# #RUN sed -i "s/'HOST': '',/'HOST': '*',/" panel/settings.py +# RUN sed -i "s/^ALLOWED_HOSTS = \[\]/ALLOWED_HOSTS = \['*'\]/" panel/settings.py +# CMD ./manage.py runserver 0.0.0.0:9080 +# EXPOSE 9080:9080 + +#RUN echo /home/orchestra/django-orchestra/ > /usr/local/lib/python3.9/dist-packages/orchestra.pth + +# TODO move this to entrypoint, with fakedata +# && su postgres bash -c 'psql -f <(zcat /home/orchestra/orchestra_db_20230907.sql)' \ + +#RUN service postgresql start \ +# && python3 manage.py setupnginx --user orchestra \ +# && su orchestra bash -c 'python3 manage.py setupcronbeat' \ +# && su orchestra bash -c 'python3 manage.py syncperiodictasks' diff --git a/docker/orchestra.entrypoint.sh b/docker/orchestra.entrypoint.sh new file mode 100755 index 0000000..721d4db --- /dev/null +++ b/docker/orchestra.entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e +set -u +#set -x + +_subs() { + key="${1}" + value="${2}" + file="${3}" + sed -i "s/^\(${key} =\).*/\1 '${value}'/" "${file}" +} + +# go to the same path as the script +cd "$(dirname ${0})" + +SECRET_KEY=${SECRET_KEY} +ALLOWED_HOSTS=${ALLOWED_HOSTS:-*} + +# override settings with env vars defined in docker +settings_file='panel/settings.py' +_subs 'ALLOWED_HOSTS' "${ALLOWED_HOSTS}" "${settings_file}" +_subs 'SECRET_KEY' "${SECRET_KEY}" "${settings_file}" + +# move the migrate thing in docker entrypoint +# inspired by https://medium.com/analytics-vidhya/django-with-docker-and-docker-compose-python-part-2-8415976470cc +#python3 manage.py migrate +expect -f ./orchestra.migrate.exp +./manage.py runserver 0.0.0.0:9080 + diff --git a/docker/orchestra.migrate.exp b/docker/orchestra.migrate.exp new file mode 100644 index 0000000..0344ef4 --- /dev/null +++ b/docker/orchestra.migrate.exp @@ -0,0 +1,61 @@ +#!/usr/bin/expect -f +# +# This Expect script was generated by autoexpect on Tue Sep 12 07:03:17 2023 +# Expect and autoexpect were both written by Don Libes, NIST. +# +# Note that autoexpect does not guarantee a working script. It +# necessarily has to guess about certain things. Two reasons a script +# might fail are: +# +# 1) timing - A surprising number of programs (rn, ksh, zsh, telnet, +# etc.) and devices discard or ignore keystrokes that arrive "too +# quickly" after prompts. If you find your new script hanging up at +# one spot, try adding a short sleep just before the previous send. +# Setting "force_conservative" to 1 (see below) makes Expect do this +# automatically - pausing briefly before sending each character. This +# pacifies every program I know of. The -c flag makes the script do +# this in the first place. The -C flag allows you to define a +# character to toggle this mode off and on. + +set force_conservative 0 ;# set to 1 to force conservative mode even if + ;# script wasn't run conservatively originally +if {$force_conservative} { + set send_slow {1 .1} + proc send {ignore arg} { + sleep .1 + exp_send -s -- $arg + } +} + +# +# 2) differing output - Some programs produce different output each time +# they run. The "date" command is an obvious example. Another is +# ftp, if it produces throughput statistics at the end of a file +# transfer. If this causes a problem, delete these patterns or replace +# them with wildcards. An alternative is to use the -p flag (for +# "prompt") which makes Expect only look for the last line of output +# (i.e., the prompt). The -P flag allows you to define a character to +# toggle this mode off and on. +# +# Read the man page for more info. +# +# -Don + + +set timeout -1 +spawn ./manage.py migrate +match_max 100000 +expect "Username: " +send -- "admin\r" +expect -exact "admin\r +Email address: " +send -- "admin@example.com\r" +expect -exact "admin@example.com\r +Password: " +send -- "admin\r" +expect -exact "\r +Password (again): " +send -- "admin\r" +expect -re "Bypass password validation and create user anyway" +send -- "y\r" +expect eof