From d04216ad795f0c8d84c542b7787621c1dab54fa0 Mon Sep 17 00:00:00 2001 From: pedro Date: Tue, 15 Oct 2024 13:09:34 +0200 Subject: [PATCH 01/26] api snapshot public_url bugfix with the following code the URL is more appropriate (it basically mirrors the user's request) before: http://localhost/device/{shortid} after http://localhost:8000/device/{shortid} --- api/views.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/views.py b/api/views.py index 175fe3e..fe68079 100644 --- a/api/views.py +++ b/api/views.py @@ -1,6 +1,5 @@ import json -from django.conf import settings from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ @@ -82,15 +81,14 @@ def NewSnapshot(request): if not annotation: return JsonResponse({'status': 'fail'}, status=200) - url = "{}://{}{}".format( - request.scheme, - settings.DOMAIN, - reverse_lazy("device:details", args=(annotation.value,)) - ) + url_args = reverse_lazy("device:details", args=(annotation.value,)) + url = request.build_absolute_uri(url_args) + response = { "status": "success", "dhid": annotation.value[:6].upper(), "url": url, + # TODO replace with public_url when available "public_url": url } return JsonResponse(response, status=200) From 69a54b92fb5318aab85f9e4429d3b9ad832cdb84 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 11:05:43 +0200 Subject: [PATCH 02/26] save snapshots on disk from api --- api/views.py | 42 +++++++++++++++++++++++++++++++++++++++--- dhub/settings.py | 1 + 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index fe68079..0ae7e04 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,8 @@ +import os import json +import shutil + +from datetime import datetime from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, redirect @@ -23,8 +27,37 @@ from api.models import Token from api.tables import TokensTable +def move_json(path_name, user): + tmp_snapshots = settings.SNAPSHOTS_DIR + path_dir = os.path.join(tmp_snapshots, user) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + def save_in_disk(data, user): - pass + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.SNAPSHOTS_DIR + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name + @csrf_exempt @@ -59,10 +92,11 @@ def NewSnapshot(request): ).first() if exist_annotation: - raise ValidationError("error: the snapshot {} exist".format(data['uuid'])) + txt = "error: the snapshot {} exist".format(data['uuid']) + return JsonResponse({'status': txt}, status=500) # Process snapshot - # save_in_disk(data, tk.user) + path_name = save_in_disk(data, tk.owner.institution.name) try: Build(data, tk.owner) @@ -91,6 +125,8 @@ def NewSnapshot(request): # TODO replace with public_url when available "public_url": url } + move_json(path_name, tk.owner.institution.name) + return JsonResponse(response, status=200) diff --git a/dhub/settings.py b/dhub/settings.py index 3b48d2c..3a55985 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -62,6 +62,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) +SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") # Application definition From fae269eb8d415f05e3c94cba446a4d629d85fd92 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 16:58:51 +0200 Subject: [PATCH 03/26] save placeholders --- api/views.py | 41 +++-------------------------------------- device/forms.py | 6 +++++- dhub/settings.py | 3 ++- evidence/forms.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/api/views.py b/api/views.py index 0ae7e04..72b5518 100644 --- a/api/views.py +++ b/api/views.py @@ -1,10 +1,9 @@ -import os import json -import shutil -from datetime import datetime +from uuid import uuid4 from django.urls import reverse_lazy +from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt @@ -16,9 +15,8 @@ from django.views.generic.edit import ( DeleteView, UpdateView, ) -from django.http import JsonResponse -from uuid import uuid4 +from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build @@ -27,39 +25,6 @@ from api.models import Token from api.tables import TokensTable -def move_json(path_name, user): - tmp_snapshots = settings.SNAPSHOTS_DIR - path_dir = os.path.join(tmp_snapshots, user) - - if os.path.isfile(path_name): - shutil.copy(path_name, path_dir) - os.remove(path_name) - - -def save_in_disk(data, user): - uuid = data.get('uuid', '') - now = datetime.now() - year = now.year - month = now.month - day = now.day - hour = now.hour - minutes = now.minute - tmp_snapshots = settings.SNAPSHOTS_DIR - - name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{user}_{uuid}.json" - path_dir = os.path.join(tmp_snapshots, user, "errors") - path_name = os.path.join(path_dir, name_file) - - if not os.path.isdir(path_dir): - os.system(f'mkdir -p {path_dir}') - - with open(path_name, 'w') as snapshot_file: - snapshot_file.write(json.dumps(data)) - - return path_name - - - @csrf_exempt def NewSnapshot(request): # Accept only posts diff --git a/device/forms.py b/device/forms.py index b2f0d3c..38d649e 100644 --- a/device/forms.py +++ b/device/forms.py @@ -1,5 +1,6 @@ from django import forms from utils.device import create_annotation, create_doc, create_index +from utils.save_snapshots import move_json, save_in_disk DEVICE_TYPES = [ @@ -55,9 +56,12 @@ class BaseDeviceFormSet(forms.BaseFormSet): doc = create_doc(row) if not commit: return doc - + + path_name = save_in_disk(doc, self.user.name) create_index(doc, self.user) create_annotation(doc, user, commit=commit) + move_json(path_name, self.user.name) + return doc diff --git a/dhub/settings.py b/dhub/settings.py index 3a55985..c307073 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -10,6 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/5.0/ref/settings/ """ +import os import xapian from pathlib import Path @@ -62,7 +63,7 @@ EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default='/tmp/app-messages') ENABLE_EMAIL = config("ENABLE_EMAIL", default=True, cast=bool) -SNAPSHOTS_DIR = config("SNAPSHOTS_DIR", default="/tmp/") +EVIDENCES_DIR = config("EVIDENCES_DIR", default=os.path.join(BASE_DIR, "db")) # Application definition diff --git a/evidence/forms.py b/evidence/forms.py index db98f2d..a47024e 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -9,6 +9,7 @@ from utils.forms import MultipleFileField from device.models import Device from evidence.parse import Build from evidence.models import Annotation +from utils.save_snapshots import move_json, save_in_disk class UploadForm(forms.Form): @@ -48,7 +49,9 @@ class UploadForm(forms.Form): return for ev in self.evidences: + path_name = save_in_disk(ev[1], user.institution.name) Build(ev[1], user) + move_json(path_name, user.institution.name) class UserTagForm(forms.Form): @@ -151,8 +154,10 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - cred.save() - create_index(doc, self.user) - return table + path_name = save_in_disk(doc, self.user.name) + cred.save() + create_index(doc, self.user) + move_json(path_name, self.user.name) + return table return From 003d224c3e2c7f499f1ecd401ba475f0b3cefe02 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:56:34 +0200 Subject: [PATCH 04/26] add save snapshots for imports and uploads --- device/forms.py | 4 ++-- evidence/forms.py | 5 +++-- utils/save_snapshots.py | 43 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 utils/save_snapshots.py diff --git a/device/forms.py b/device/forms.py index 38d649e..6c4d6e2 100644 --- a/device/forms.py +++ b/device/forms.py @@ -57,10 +57,10 @@ class BaseDeviceFormSet(forms.BaseFormSet): if not commit: return doc - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") create_index(doc, self.user) create_annotation(doc, user, commit=commit) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return doc diff --git a/evidence/forms.py b/evidence/forms.py index a47024e..cf90a71 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -154,10 +154,11 @@ class ImportForm(forms.Form): if commit: for doc, cred in table: - path_name = save_in_disk(doc, self.user.name) + path_name = save_in_disk(doc, self.user.institution.name, place="placeholder") + cred.save() create_index(doc, self.user) - move_json(path_name, self.user.name) + move_json(path_name, self.user.institution.name, place="placeholder") return table return diff --git a/utils/save_snapshots.py b/utils/save_snapshots.py new file mode 100644 index 0000000..8c02f06 --- /dev/null +++ b/utils/save_snapshots.py @@ -0,0 +1,43 @@ +import os +import json +import shutil + +from datetime import datetime +from django.conf import settings + + +def move_json(path_name, user, place="snapshots"): + if place != "snapshots": + place = "placeholders" + + tmp_snapshots = settings.EVIDENCES_DIR + path_dir = os.path.join(tmp_snapshots, user, place) + + if os.path.isfile(path_name): + shutil.copy(path_name, path_dir) + os.remove(path_name) + + +def save_in_disk(data, user, place="snapshots"): + uuid = data.get('uuid', '') + now = datetime.now() + year = now.year + month = now.month + day = now.day + hour = now.hour + minutes = now.minute + tmp_snapshots = settings.EVIDENCES_DIR + if place != "snapshots": + place = "placeholders" + + name_file = f"{year}-{month}-{day}-{hour}-{minutes}_{uuid}.json" + path_dir = os.path.join(tmp_snapshots, user, place, "errors") + path_name = os.path.join(path_dir, name_file) + + if not os.path.isdir(path_dir): + os.system(f'mkdir -p {path_dir}') + + with open(path_name, 'w') as snapshot_file: + snapshot_file.write(json.dumps(data)) + + return path_name From 2f9b61667d38dd84a2c0e4dd9dc5568a0054ba31 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 15 Oct 2024 17:57:45 +0200 Subject: [PATCH 05/26] remove pdbs --- login/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/login/views.py b/login/views.py index 5bf48a3..be6b009 100644 --- a/login/views.py +++ b/login/views.py @@ -65,7 +65,6 @@ class PasswordResetView(auth_views.PasswordResetView): success_url = reverse_lazy('login:password_reset_done') def form_valid(self, form): - import pdb; pdb.set_trace() try: response = super().form_valid(form) return response From 090ecc275fbf6b7fc876268db4492b365f8dc4c3 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:17:35 +0200 Subject: [PATCH 06/26] fix bug of tag --- dashboard/views.py | 12 ++---- device/models.py | 102 +++++++++++++++++++++++++++++++++++++++++---- evidence/models.py | 22 ++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/dashboard/views.py b/dashboard/views.py index b34e2a6..f056523 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -69,23 +69,17 @@ class SearchView(InventaryMixin): if not matches.size(): return self.search_hids(query, offset, limit) - annotations = [] + devices = [] for x in matches: - annotations.extend(self.get_annotations(x)) + devices.append(self.get_annotations(x)) - devices = [Device(id=x) for x in set(annotations)] count = matches.size() return devices, count def get_annotations(self, xp): snap = xp.document.get_data() uuid = json.loads(snap).get('uuid') - - return Annotation.objects.filter( - type=Annotation.Type.SYSTEM, - owner=self.request.user.institution, - uuid=uuid - ).values_list("value", flat=True).distinct() + return Device.get_annotation_from_uuid(uuid, self.request.user.institution) def search_hids(self, query, offset, limit): qry = Q() diff --git a/device/models.py b/device/models.py index 926e447..6169f2e 100644 --- a/device/models.py +++ b/device/models.py @@ -118,9 +118,32 @@ class Device: def get_unassigned(cls, institution, offset=0, limit=None): sql = """ - SELECT DISTINCT t1.value from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type} + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, @@ -144,18 +167,83 @@ class Device: def get_unassigned_count(cls, institution): sql = """ - SELECT count(DISTINCT t1.value) from evidence_annotation as t1 - left join lot_devicelot as t2 on t1.value = t2.device_id - where t2.device_id is null and owner_id=={institution} and type=={type}; + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + ) + SELECT + COUNT(DISTINCT value) + FROM + RankedAnnotations + WHERE + row_num = 1 """.format( institution=institution.id, type=Annotation.Type.SYSTEM, ) - with connection.cursor() as cursor: cursor.execute(sql) return cursor.fetchall()[0][0] + @classmethod + def get_annotation_from_uuid(cls, uuid, institution): + sql = """ + WITH RankedAnnotations AS ( + SELECT + t1.value, + t1.key, + ROW_NUMBER() OVER ( + PARTITION BY t1.uuid + ORDER BY + CASE + WHEN t1.key = 'CUSTOM_ID' THEN 1 + WHEN t1.key = 'hidalgo1' THEN 2 + ELSE 3 + END, + t1.created DESC + ) AS row_num + FROM evidence_annotation AS t1 + LEFT JOIN lot_devicelot AS t2 ON t1.value = t2.device_id + WHERE t2.device_id IS NULL + AND t1.owner_id = {institution} + AND t1.type = {type} + AND t1.uuid = '{uuid}' + ) + SELECT DISTINCT + value + FROM + RankedAnnotations + WHERE + row_num = 1; + """.format( + uuid=uuid.replace("-", ""), + institution=institution.id, + type=Annotation.Type.SYSTEM, + ) + + annotations = [] + with connection.cursor() as cursor: + cursor.execute(sql) + annotations = cursor.fetchall() + + return cls(id=annotations[0][0]) + @property def is_websnapshot(self): if not self.last_evidence: diff --git a/evidence/models.py b/evidence/models.py index 667945f..ffc0d84 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -126,8 +126,30 @@ class Evidence: return Annotation.objects.filter( owner=user.institution, type=Annotation.Type.SYSTEM, + key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() + # annotations = Annotation.objects.filter( + # owner=user.institution, + # type=Annotation.Type.SYSTEM + # ) + + # annotations = annotations.annotate( + # priority=models.Case( + # models.When(key="CUSTOM_ID", then=1), + # models.When(key="hidalgo1", then=2), + # default=3, + # output_field=models.IntegerField(), + # ) + # ) + + # # sqlite not soport distinct('uuid') + # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # # 'uuid').values_list("uuid", flat=True) + # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( + # ).values_list("uuid", flat=True) + # return set(filtered_uuids) + def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From c422eaddebb9e6d6bfab9064ff087c2614bc0c7e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 14:19:09 +0200 Subject: [PATCH 07/26] clean code comment --- evidence/models.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/evidence/models.py b/evidence/models.py index ffc0d84..354cb89 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -129,27 +129,6 @@ class Evidence: key="hidalgo1", ).order_by("-created").values_list("uuid", flat=True).distinct() - # annotations = Annotation.objects.filter( - # owner=user.institution, - # type=Annotation.Type.SYSTEM - # ) - - # annotations = annotations.annotate( - # priority=models.Case( - # models.When(key="CUSTOM_ID", then=1), - # models.When(key="hidalgo1", then=2), - # default=3, - # output_field=models.IntegerField(), - # ) - # ) - - # # sqlite not soport distinct('uuid') - # # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # # 'uuid').values_list("uuid", flat=True) - # filtered_uuids = annotations.order_by('uuid', 'priority', '-created').distinct( - # ).values_list("uuid", flat=True) - # return set(filtered_uuids) - def set_components(self): snapshot = ParseSnapshot(self.doc).snapshot_json self.components = snapshot['components'] From d8b6d3ded61df6f5c6d769f31cedaf13dd167730 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 16 Oct 2024 17:40:00 +0200 Subject: [PATCH 08/26] fix hid in details of device --- device/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/device/models.py b/device/models.py index 6169f2e..1f43476 100644 --- a/device/models.py +++ b/device/models.py @@ -90,9 +90,11 @@ class Device: def get_hids(self): annotations = self.get_annotations() + algos = list(ALGOS.keys()) + algos.append('CUSTOM_ID') self.hids = list(set(annotations.filter( type=Annotation.Type.SYSTEM, - key__in=ALGOS.keys(), + key__in=algos, ).values_list("value", flat=True))) def get_evidences(self): From 3d10217599d14ce44e1d1b11fb83197e018e2b10 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 20:39:57 +0200 Subject: [PATCH 09/26] docker-reset: remove better the db now that in db dir there are snapshots stored --- docker-reset.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-reset.sh b/docker-reset.sh index cdbb971..bf14db2 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -9,11 +9,13 @@ set -u set -x main() { + cd "$(dirname "${0}")" + if [ "${DETACH:-}" ]; then detach_arg='-d' fi # remove old database - sudo rm -vf db/* + sudo rm -vfr ./db/* docker compose down -v docker compose build docker compose up ${detach_arg:-} From 0e8607d93ecf45f3573220e97144e91a619c921a Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:09:16 +0200 Subject: [PATCH 10/26] docker: make default user admin --- docker/devicehub-django.entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 920cdf0..9361fdb 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -24,9 +24,10 @@ deploy() { INIT_ORG="${INIT_ORG:-example-org}" INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" + ADMIN='True' ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" From f757c5835656b15c8daa32dee36e26c85d8009d5 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:14:22 +0200 Subject: [PATCH 11/26] snapshot processing: improve warning msg add comment TODO that system annotation should happen --- evidence/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evidence/parse.py b/evidence/parse.py index fa0903c..dfc5a1f 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -113,7 +113,8 @@ class Build: # mac = get_mac2(hwinfo_raw) or "" mac = get_mac(lshw) or "" if not mac: - print("WARNING!! No there are MAC address") + print(f"WARNING: Could not retrieve MAC address in snapshot {snapshot['uuid']}" ) + # TODO generate system annotation for that snapshot else: print(f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}") From 67869bc6f55e45de170f8ae2222def8fe90d053f Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 21:54:54 +0200 Subject: [PATCH 12/26] add predefined_token as an alternative to the randomly generated, which is also possible when no predefined_token is defined also update .env.example vars --- .env.example | 19 +++++++++++++++++++ docker-compose.yml | 1 + docker/devicehub-django.entrypoint.sh | 3 ++- user/management/commands/add_user.py | 12 +++++++++--- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index cf39121..1495f9d 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,21 @@ DOMAIN=localhost DEMO=false + +STATIC_ROOT=/tmp/static/ +MEDIA_ROOT=/tmp/media/ +ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1, +DOMAIN=localhost +DEBUG=True +EMAIL_HOST="mail.example.org" +EMAIL_HOST_USER="fillme_noreply" +EMAIL_HOST_PASSWORD="fillme_passwd" +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" +EMAIL_FILE_PATH="/tmp/app-messages" +ENABLE_EMAIL=false +COMMIT=$(git log --format="%H %ad" --date=iso -n 1) +PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' +# TODO review these vars +#SNAPSHOTS_DIR=/path/to/TODO +#EVIDENCES_DIR=/path/to/TODO diff --git a/docker-compose.yml b/docker-compose.yml index ee59640..7d144bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - DEMO=${DEMO:-n} + - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django ports: diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 9361fdb..e988d4c 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -25,9 +25,10 @@ deploy() { INIT_USER="${INIT_USER:-user@example.org}" INIT_PASSWD="${INIT_PASSWD:-1234}" ADMIN='True' + PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}" ./manage.py add_institution "${INIT_ORG}" # TODO: one error on add_user, and you don't add user anymore - ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" + ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" if [ "${DEMO:-}" ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" diff --git a/user/management/commands/add_user.py b/user/management/commands/add_user.py index 60f3dd2..6593fee 100644 --- a/user/management/commands/add_user.py +++ b/user/management/commands/add_user.py @@ -17,15 +17,17 @@ class Command(BaseCommand): parser.add_argument('email', type=str, help='email') parser.add_argument('password', type=str, help='password') parser.add_argument('is_admin', nargs='?', default=False, type=str, help='is admin') + parser.add_argument('predefined_token', nargs='?', default='', type=str, help='predefined token') def handle(self, *args, **kwargs): email = kwargs['email'] password = kwargs['password'] is_admin = kwargs['is_admin'] + predefined_token = kwargs['predefined_token'] institution = Institution.objects.get(name=kwargs['institution']) - self.create_user(institution, email, password, is_admin) + self.create_user(institution, email, password, is_admin, predefined_token) - def create_user(self, institution, email, password, is_admin): + def create_user(self, institution, email, password, is_admin, predefined_token): self.u = User.objects.create( institution=institution, email=email, @@ -34,6 +36,10 @@ class Command(BaseCommand): ) self.u.set_password(password) self.u.save() - token = uuid4() + if predefined_token: + token = predefined_token + else: + token = uuid4() + Token.objects.create(token=token, owner=self.u) print(f"TOKEN: {token}") From bf47d4bc5d20a2ed61cfab04540494d37d789577 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:05:09 +0200 Subject: [PATCH 13/26] docker: bugfix wrong detection of demo --- docker/devicehub-django.entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e988d4c..2c7c77b 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" ]; then + if [ "${DEMO:-}" = 'y' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From 425b032273bc4f6d77ff0f4e10df3bec0740cb80 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:07:40 +0200 Subject: [PATCH 14/26] docker: use demo true/false instead of y/n --- docker-compose.yml | 2 +- docker/devicehub-django.entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d144bd..b451358 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - DEMO=${DEMO:-n} + - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: - .:/opt/devicehub-django diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 2c7c77b..452b32d 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -30,7 +30,7 @@ deploy() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DEMO:-}" = 'y' ]; then + if [ "${DEMO:-}" = 'true' ]; then ./manage.py up_snapshots example/snapshots/ "${INIT_USER}" fi fi From c49644ffbd80990264c43e16d3d39c70bd82e1e1 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 16 Oct 2024 22:48:42 +0200 Subject: [PATCH 15/26] add ALLOWED_HOSTS in docker compose --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index b451358..9ea039c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From 0b55d34d74ce092de75b9285d62b946b0942d809 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 18:38:57 +0200 Subject: [PATCH 16/26] locale in settings --- dhub/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhub/settings.py b/dhub/settings.py index c307073..e109072 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -164,6 +164,11 @@ USE_I18N = True USE_TZ = True +USE_L10N = True +LANGUAGES = [ + ('es', 'Spanish'), + ('en', 'English'), +] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.0/howto/static-files/ From e6141bcc7fabf528e82f812b895773a3f416972f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 17 Oct 2024 19:08:00 +0200 Subject: [PATCH 17/26] more debug in the error answer from api --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 72b5518..79f3592 100644 --- a/api/views.py +++ b/api/views.py @@ -65,8 +65,8 @@ def NewSnapshot(request): try: Build(data, tk.owner) - except Exception: - return JsonResponse({'status': 'fail'}, status=200) + except Exception as err: + return JsonResponse({'status': f"fail: {err}"}, status=500) annotation = Annotation.objects.filter( uuid=data['uuid'], From b7c4926c390f8d91375e18bf7a655d55a992739d Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 09:52:09 +0200 Subject: [PATCH 18/26] add commit_id in settings --- dashboard/mixins.py | 2 ++ dhub/settings.py | 1 + 2 files changed, 3 insertions(+) diff --git a/dashboard/mixins.py b/dashboard/mixins.py index aa63d36..af01a1b 100644 --- a/dashboard/mixins.py +++ b/dashboard/mixins.py @@ -1,4 +1,5 @@ from django.urls import resolve +from django.conf import settings from django.shortcuts import get_object_or_404, redirect, Http404 from django.utils.translation import gettext_lazy as _ from django.core.exceptions import PermissionDenied @@ -32,6 +33,7 @@ class DashboardView(LoginRequiredMixin): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context.update({ + "commit_id": settings.COMMIT, 'title': self.title, 'subtitle': self.subtitle, 'breadcrumb': self.breadcrumb, diff --git a/dhub/settings.py b/dhub/settings.py index e109072..b96710e 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -213,3 +213,4 @@ LOGGING = { SNAPSHOT_PATH="/tmp/" DATA_UPLOAD_MAX_NUMBER_FILES = 1000 +COMMIT = config('COMMIT', default='') From 3d2cb7d184c381dc0e0cbdb9e97ca18015328f6a Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:20:23 +0200 Subject: [PATCH 19/26] docker: improve debug --- docker/devicehub-django.entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 452b32d..e5773f3 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + if [ "${DEBUG:-}" = 'true' ]; then + ./manage.py print_settings + fi + # detect if existing deployment (TODO only works with sqlite) if [ -f "${program_dir}/db/db.sqlite3" ]; then echo "INFO: detected EXISTING deployment" @@ -38,7 +42,7 @@ deploy() { runserver() { PORT="${PORT:-8000}" - if [ "${DEBUG:-}" ]; then + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py runserver 0.0.0.0:${PORT} else # TODO From d995db8181361a7b8faf7466c252086baff55d02 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:24:40 +0200 Subject: [PATCH 20/26] settings: improve assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index b96710e..80d9f3c 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, "DOMAIN is not ALLOWED_HOST" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 096b83da812ef66105d88382b0f69c19d0144ed6 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:26:38 +0200 Subject: [PATCH 21/26] settings: bugfix assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index 80d9f3c..c580e6b 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not ALLOWED_HOST ${ALLOWED_HOST}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 1d9d100fae1d490c55c2b89f3470c0908a4d23f6 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:27:27 +0200 Subject: [PATCH 22/26] settings: bugfix format assert error msg on DOMAIN --- dhub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhub/settings.py b/dhub/settings.py index c580e6b..891b714 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -36,7 +36,7 @@ assert DOMAIN not in [None, ''], "DOMAIN var is MANDATORY" print("DOMAIN: " + DOMAIN) ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=DOMAIN, cast=Csv()) -assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN ${DOMAIN} is not in ALLOWED_HOSTS ${ALLOWED_HOSTS}" +assert DOMAIN in ALLOWED_HOSTS, f"DOMAIN {DOMAIN} is not in ALLOWED_HOSTS {ALLOWED_HOSTS}" CSRF_TRUSTED_ORIGINS = config('CSRF_TRUSTED_ORIGINS', default=f'https://{DOMAIN}', cast=Csv()) From 9f05dee6a7656179e0ec396732ca651e61434a19 Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:28:02 +0200 Subject: [PATCH 23/26] docker-compose: bugfix DOMAIN vs ALLOWED_HOSTS --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9ea039c..d376313 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: - DEBUG=true - DOMAIN=${DOMAIN:-localhost} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-$DOMAIN} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} volumes: From 4cf164d4bce2656c3b388435ff8dafa7eae7fa8b Mon Sep 17 00:00:00 2001 From: pedro Date: Fri, 18 Oct 2024 10:45:08 +0200 Subject: [PATCH 24/26] add commit footer for docker --- .env.example | 1 - docker/devicehub-django.entrypoint.sh | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 1495f9d..37ed087 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,6 @@ EMAIL_USE_TLS=True EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend" EMAIL_FILE_PATH="/tmp/app-messages" ENABLE_EMAIL=false -COMMIT=$(git log --format="%H %ad" --date=iso -n 1) PREDEFINED_TOKEN='5018dd65-9abd-4a62-8896-80f34ac66150' # TODO review these vars #SNAPSHOTS_DIR=/path/to/TODO diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index e5773f3..bdf1008 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -12,6 +12,10 @@ check_app_is_there() { } deploy() { + # TODO this is weird, find better workaround + git config --global --add safe.directory /opt/devicehub-django + export COMMIT=$(git log --format="%H %ad" --date=iso -n 1) + if [ "${DEBUG:-}" = 'true' ]; then ./manage.py print_settings fi From c60a9dd89d368e9dcf849a4dc470c7a57545d8e4 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 12:03:26 +0200 Subject: [PATCH 25/26] commit in login page --- login/templates/login.html | 1 + login/views.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/login/templates/login.html b/login/templates/login.html index 97ba4cb..a82882e 100644 --- a/login/templates/login.html +++ b/login/templates/login.html @@ -42,4 +42,5 @@ + {% endblock %} diff --git a/login/views.py b/login/views.py index be6b009..beb0368 100644 --- a/login/views.py +++ b/login/views.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.urls import reverse_lazy from django.contrib.auth import views as auth_views from django.contrib.auth import login as auth_login @@ -17,7 +18,7 @@ class LoginView(auth_views.LoginView): extra_context = { 'title': _('Login'), 'success_url': reverse_lazy('dashboard:unassigned_devices'), - # 'commit_id': settings.COMMIT, + 'commit_id': settings.COMMIT, } def get(self, request, *args, **kwargs): From d98e8e2d48597333e06bf781ac27588fd792f0ec Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Oct 2024 15:24:42 +0200 Subject: [PATCH 26/26] resolve conficts --- api/views.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api/views.py b/api/views.py index 79f3592..69da2a2 100644 --- a/api/views.py +++ b/api/views.py @@ -7,9 +7,7 @@ from django.http import JsonResponse from django.shortcuts import get_object_or_404, redirect from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt -from django.core.exceptions import ValidationError from django_tables2 import SingleTableView -from django.views.generic.base import View from django.views.generic.edit import ( CreateView, DeleteView, @@ -20,7 +18,6 @@ from utils.save_snapshots import move_json, save_in_disk from dashboard.mixins import DashboardView from evidence.models import Annotation from evidence.parse import Build -from user.models import User from api.models import Token from api.tables import TokensTable @@ -78,7 +75,7 @@ def NewSnapshot(request): if not annotation: - return JsonResponse({'status': 'fail'}, status=200) + return JsonResponse({'status': 'fail'}, status=500) url_args = reverse_lazy("device:details", args=(annotation.value,)) url = request.build_absolute_uri(url_args)