From 9857891b637f6e9993138a8559fbec2af49990a2 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 15 Nov 2024 18:56:11 +0100 Subject: [PATCH 01/80] first base for dpp --- dhub/settings.py | 1 + dpp/__init__.py | 0 dpp/admin.py | 3 ++ dpp/api_dlt.py | 96 ++++++++++++++++++++++++++++++++++ dpp/apps.py | 6 +++ dpp/migrations/0001_initial.py | 90 +++++++++++++++++++++++++++++++ dpp/migrations/__init__.py | 0 dpp/models.py | 30 +++++++++++ dpp/tests.py | 3 ++ dpp/views.py | 3 ++ 10 files changed, 232 insertions(+) create mode 100644 dpp/__init__.py create mode 100644 dpp/admin.py create mode 100644 dpp/api_dlt.py create mode 100644 dpp/apps.py create mode 100644 dpp/migrations/0001_initial.py create mode 100644 dpp/migrations/__init__.py create mode 100644 dpp/models.py create mode 100644 dpp/tests.py create mode 100644 dpp/views.py diff --git a/dhub/settings.py b/dhub/settings.py index 0dc3d84..4b5c366 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -89,6 +89,7 @@ INSTALLED_APPS = [ "dashboard", "admin", "api", + "dpp", ] diff --git a/dpp/__init__.py b/dpp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dpp/admin.py b/dpp/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/dpp/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/dpp/api_dlt.py b/dpp/api_dlt.py new file mode 100644 index 0000000..6a1317a --- /dev/null +++ b/dpp/api_dlt.py @@ -0,0 +1,96 @@ +from ereuseapi.methods import API + + +def connect_api(): + + if not session.get('token_dlt'): + return + + token_dlt = session.get('token_dlt') + api_dlt = app.config.get('API_DLT') + + return API(api_dlt, token_dlt, "ethereum") + +def register_dlt(): + api = self.connect_api() + if not api: + return + + snapshot = [x for x in self.actions if x.t == 'Snapshot'] + if not snapshot: + return + snapshot = snapshot[0] + from ereuse_devicehub.modules.dpp.models import ALGORITHM + from ereuse_devicehub.resources.enums import StatusCode + cny_a = 1 + while cny_a: + api = self.connect_api() + result = api.register_device( + self.chid, + ALGORITHM, + snapshot.phid_dpp, + app.config.get('ID_FEDERATED') + ) + try: + assert result['Status'] == StatusCode.Success.value + assert result['Data']['data']['timestamp'] + cny_a = 0 + except Exception: + if result.get("Data") != "Device already exists": + logger.error("API return: %s", result) + time.sleep(10) + else: + cny_a = 0 + + + register_proof(result) + + if app.config.get('ID_FEDERATED'): + cny = 1 + while cny: + try: + api.add_service( + self.chid, + 'DeviceHub', + app.config.get('ID_FEDERATED'), + 'Inventory service', + 'Inv', + ) + cny = 0 + except Exception: + time.sleep(10) + +def register_proof(self, result): + from ereuse_devicehub.modules.dpp.models import PROOF_ENUM, Proof + from ereuse_devicehub.resources.enums import StatusCode + + if result['Status'] == StatusCode.Success.value: + timestamp = result.get('Data', {}).get('data', {}).get('timestamp') + + if not timestamp: + return + + snapshot = [x for x in self.actions if x.t == 'Snapshot'] + if not snapshot: + return + snapshot = snapshot[0] + + d = { + "type": PROOF_ENUM['Register'], + "device": self, + "action": snapshot, + "timestamp": timestamp, + "issuer_id": g.user.id, + "documentId": snapshot.id, + "documentSignature": snapshot.phid_dpp, + "normalizeDoc": snapshot.json_hw, + } + proof = Proof(**d) + db.session.add(proof) + + if not hasattr(self, 'components'): + return + + for c in self.components: + if isinstance(c, DataStorage): + c.register_dlt() diff --git a/dpp/apps.py b/dpp/apps.py new file mode 100644 index 0000000..758fbb0 --- /dev/null +++ b/dpp/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class DppConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "dpp" diff --git a/dpp/migrations/0001_initial.py b/dpp/migrations/0001_initial.py new file mode 100644 index 0000000..0f10686 --- /dev/null +++ b/dpp/migrations/0001_initial.py @@ -0,0 +1,90 @@ +# Generated by Django 5.0.6 on 2024-11-15 18:55 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("user", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Dpp", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("timestamp", models.IntegerField()), + ("key", models.CharField(max_length=256)), + ("uuid", models.UUIDField()), + ("signature", models.CharField(max_length=256)), + ("normalizeDoc", models.TextField()), + ("type", models.CharField(max_length=256)), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="user.institution", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.CreateModel( + name="Proof", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("timestamp", models.IntegerField()), + ("uuid", models.UUIDField()), + ("signature", models.CharField(max_length=256)), + ("normalizeDoc", models.TextField()), + ("type", models.CharField(max_length=256)), + ("action", models.CharField(max_length=256)), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="user.institution", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/dpp/migrations/__init__.py b/dpp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dpp/models.py b/dpp/models.py new file mode 100644 index 0000000..d46f7de --- /dev/null +++ b/dpp/models.py @@ -0,0 +1,30 @@ +from django.db import models +from user.models import User, Institution +from utils.constants import STR_EXTEND_SIZE +# Create your models here. + + +class Proof(models.Model): + timestamp = models.IntegerField() + uuid = models.UUIDField() + signature = models.CharField(max_length=STR_EXTEND_SIZE) + normalizeDoc = models.TextField() + type = models.CharField(max_length=STR_EXTEND_SIZE) + action = models.CharField(max_length=STR_EXTEND_SIZE) + owner = models.ForeignKey(Institution, on_delete=models.CASCADE) + user = models.ForeignKey( + User, on_delete=models.SET_NULL, null=True, blank=True) + + +class Dpp(models.Model): + timestamp = models.IntegerField() + key = models.CharField(max_length=STR_EXTEND_SIZE) + uuid = models.UUIDField() + signature = models.CharField(max_length=STR_EXTEND_SIZE) + normalizeDoc = models.TextField() + type = models.CharField(max_length=STR_EXTEND_SIZE) + owner = models.ForeignKey(Institution, on_delete=models.CASCADE) + user = models.ForeignKey( + User, on_delete=models.SET_NULL, null=True, blank=True) + + diff --git a/dpp/tests.py b/dpp/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/dpp/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/dpp/views.py b/dpp/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/dpp/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. From 0d574cae63070d7e54c947df037faf2a486bb702 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 18 Nov 2024 19:37:08 +0100 Subject: [PATCH 02/80] register device and dpp in dlt and dpp api --- device/templates/details.html | 19 ++++ device/views.py | 3 + dhub/urls.py | 1 + dpp/api_dlt.py | 183 +++++++++++++++++++++------------ dpp/migrations/0001_initial.py | 42 +------- dpp/models.py | 19 +--- dpp/urls.py | 8 ++ dpp/views.py | 37 ++++++- evidence/models.py | 7 ++ evidence/parse.py | 15 ++- evidence/xapian.py | 12 ++- 11 files changed, 218 insertions(+), 128 deletions(-) create mode 100644 dpp/urls.py diff --git a/device/templates/details.html b/device/templates/details.html index 137a4f6..9783057 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -29,6 +29,9 @@ + @@ -229,6 +232,22 @@ {% endfor %} + +
+
{% trans 'List of dpps' %}
+
+ {% for d in dpps %} +
+
+ {{ d.timestamp }} +
+

+ {{ d.signature }} +

+
+ {% endfor %} +
+
{% endblock %} diff --git a/device/views.py b/device/views.py index 319f8cf..12fb244 100644 --- a/device/views.py +++ b/device/views.py @@ -14,6 +14,7 @@ from django.views.generic.base import TemplateView from dashboard.mixins import DashboardView, Http403 from evidence.models import Annotation from lot.models import LotTag +from dpp.models import Proof from device.models import Device from device.forms import DeviceFormSet @@ -103,10 +104,12 @@ class DetailsView(DashboardView, TemplateView): context = super().get_context_data(**kwargs) self.object.initial() lot_tags = LotTag.objects.filter(owner=self.request.user.institution) + dpps = Proof.objects.filter(uuid_in=self.object.uuids) context.update({ 'object': self.object, 'snapshot': self.object.get_last_evidence(), 'lot_tags': lot_tags, + 'dpps': dpps, }) return context diff --git a/dhub/urls.py b/dhub/urls.py index fba9bd3..c50da55 100644 --- a/dhub/urls.py +++ b/dhub/urls.py @@ -27,4 +27,5 @@ urlpatterns = [ path("user/", include("user.urls")), path("lot/", include("lot.urls")), path('api/', include('api.urls')), + path('dpp/', include('dpp.urls')), ] diff --git a/dpp/api_dlt.py b/dpp/api_dlt.py index 6a1317a..6bf95c0 100644 --- a/dpp/api_dlt.py +++ b/dpp/api_dlt.py @@ -1,38 +1,106 @@ +import time +import logging + +from django.conf import settings from ereuseapi.methods import API +from dpp.models import Proof, Dpp + + +logger = logging.getLogger('django') + + +# """The code of the status response of api dlt.""" +STATUS_CODE = { + "Success": 201, + "Notwork": 400 +} + + +ALGORITHM = "sha3-256" + + +PROOF_TYPE = { + 'Register': 'Register', + 'IssueDPP': 'IssueDPP', + 'proof_of_recycling': 'proof_of_recycling', + 'Erase': 'Erase', + 'EWaste': 'EWaste', +} + def connect_api(): - if not session.get('token_dlt'): + if not settings.get('TOKEN_DLT'): return - token_dlt = session.get('token_dlt') - api_dlt = app.config.get('API_DLT') + token_dlt = settings.get('TOKEN_DLT') + api_dlt = settings.get('API_DLT') return API(api_dlt, token_dlt, "ethereum") -def register_dlt(): - api = self.connect_api() + +def register_dlt(chid, phid, proof_type=None): + api = connect_api() if not api: return - snapshot = [x for x in self.actions if x.t == 'Snapshot'] - if not snapshot: + if proof_type: + return api.generate_proof( + chid, + ALGORITHM, + phid, + proof_type, + settings.get('ID_FEDERATED') + ) + + return api.register_device( + chid, + ALGORITHM, + phid, + settings.get('ID_FEDERATED') + ) + + +def issuer_dpp_dlt(dpp): + phid = dpp.split(":")[0] + api = connect_api() + if not api: return - snapshot = snapshot[0] - from ereuse_devicehub.modules.dpp.models import ALGORITHM - from ereuse_devicehub.resources.enums import StatusCode + + return api.issue_passport( + dpp, + ALGORITHM, + phid, + settings.get('ID_FEDERATED') + ) + + + +def save_proof(signature, ev_uuid, result, proof_type, user): + if result['Status'] == STATUS_CODE.get("Success"): + timestamp = result.get('Data', {}).get('data', {}).get('timestamp') + + if not timestamp: + return + + d = { + "type": proof_type, + "timestamp": timestamp, + "issuer": user.institution.id, + "user": user, + "uuid": ev_uuid, + "signature": signature, + } + Proof.objects.create(**d) + + +def register_device_dlt(chid, phid, ev_uuid, user): cny_a = 1 while cny_a: - api = self.connect_api() - result = api.register_device( - self.chid, - ALGORITHM, - snapshot.phid_dpp, - app.config.get('ID_FEDERATED') - ) + result = register_dlt(chid, phid) try: - assert result['Status'] == StatusCode.Success.value + assert result['Status'] == STATUS_CODE.get("Success") assert result['Data']['data']['timestamp'] cny_a = 0 except Exception: @@ -42,55 +110,42 @@ def register_dlt(): else: cny_a = 0 + save_proof(phid, ev_uuid, result, PROOF_TYPE['Register'], user) - register_proof(result) - if app.config.get('ID_FEDERATED'): - cny = 1 - while cny: - try: - api.add_service( - self.chid, - 'DeviceHub', - app.config.get('ID_FEDERATED'), - 'Inventory service', - 'Inv', - ) - cny = 0 - except Exception: - time.sleep(10) + # TODO is neccesary? + # if settings.get('ID_FEDERATED'): + # cny = 1 + # while cny: + # try: + # api.add_service( + # chid, + # 'DeviceHub', + # settings.get('ID_FEDERATED'), + # 'Inventory service', + # 'Inv', + # ) + # cny = 0 + # except Exception: + # time.sleep(10) -def register_proof(self, result): - from ereuse_devicehub.modules.dpp.models import PROOF_ENUM, Proof - from ereuse_devicehub.resources.enums import StatusCode - if result['Status'] == StatusCode.Success.value: - timestamp = result.get('Data', {}).get('data', {}).get('timestamp') - - if not timestamp: - return - - snapshot = [x for x in self.actions if x.t == 'Snapshot'] - if not snapshot: - return - snapshot = snapshot[0] - - d = { - "type": PROOF_ENUM['Register'], - "device": self, - "action": snapshot, - "timestamp": timestamp, - "issuer_id": g.user.id, - "documentId": snapshot.id, - "documentSignature": snapshot.phid_dpp, - "normalizeDoc": snapshot.json_hw, - } - proof = Proof(**d) - db.session.add(proof) - - if not hasattr(self, 'components'): +def register_passport_dlt(chid, phid, ev_uuid, user): + dpp = "{chid}:{phid}".format(chid=chid, phid=phid) + if Proof.objects.filter(signature=dpp, type=PROOF_TYPE['IssueDPP']).exists(): return - for c in self.components: - if isinstance(c, DataStorage): - c.register_dlt() + cny_a = 1 + while cny_a: + try: + result = issuer_dpp_dlt(dpp) + cny_a = 0 + except Exception as err: + logger.error("ERROR API issue passport return: %s", err) + time.sleep(10) + + if result['Status'] is not STATUS_CODE.get("Success"): + logger.error("ERROR API issue passport return: %s", result) + return + + save_proof(phid, ev_uuid, result, PROOF_TYPE['IssueDPP'], user) diff --git a/dpp/migrations/0001_initial.py b/dpp/migrations/0001_initial.py index 0f10686..71e1b75 100644 --- a/dpp/migrations/0001_initial.py +++ b/dpp/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.6 on 2024-11-15 18:55 +# Generated by Django 5.0.6 on 2024-11-18 14:29 import django.db.models.deletion from django.conf import settings @@ -15,42 +15,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.CreateModel( - name="Dpp", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("timestamp", models.IntegerField()), - ("key", models.CharField(max_length=256)), - ("uuid", models.UUIDField()), - ("signature", models.CharField(max_length=256)), - ("normalizeDoc", models.TextField()), - ("type", models.CharField(max_length=256)), - ( - "owner", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="user.institution", - ), - ), - ( - "user", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ], - ), migrations.CreateModel( name="Proof", fields=[ @@ -66,11 +30,9 @@ class Migration(migrations.Migration): ("timestamp", models.IntegerField()), ("uuid", models.UUIDField()), ("signature", models.CharField(max_length=256)), - ("normalizeDoc", models.TextField()), ("type", models.CharField(max_length=256)), - ("action", models.CharField(max_length=256)), ( - "owner", + "issuer", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, to="user.institution", diff --git a/dpp/models.py b/dpp/models.py index d46f7de..5502bcf 100644 --- a/dpp/models.py +++ b/dpp/models.py @@ -5,26 +5,11 @@ from utils.constants import STR_EXTEND_SIZE class Proof(models.Model): + ## The signature can be a phid or dpp depending of type of Proof timestamp = models.IntegerField() uuid = models.UUIDField() signature = models.CharField(max_length=STR_EXTEND_SIZE) - normalizeDoc = models.TextField() type = models.CharField(max_length=STR_EXTEND_SIZE) - action = models.CharField(max_length=STR_EXTEND_SIZE) - owner = models.ForeignKey(Institution, on_delete=models.CASCADE) + issuer = models.ForeignKey(Institution, on_delete=models.CASCADE) user = models.ForeignKey( User, on_delete=models.SET_NULL, null=True, blank=True) - - -class Dpp(models.Model): - timestamp = models.IntegerField() - key = models.CharField(max_length=STR_EXTEND_SIZE) - uuid = models.UUIDField() - signature = models.CharField(max_length=STR_EXTEND_SIZE) - normalizeDoc = models.TextField() - type = models.CharField(max_length=STR_EXTEND_SIZE) - owner = models.ForeignKey(Institution, on_delete=models.CASCADE) - user = models.ForeignKey( - User, on_delete=models.SET_NULL, null=True, blank=True) - - diff --git a/dpp/urls.py b/dpp/urls.py new file mode 100644 index 0000000..9ada14e --- /dev/null +++ b/dpp/urls.py @@ -0,0 +1,8 @@ +from django.urls import path +from dpp import views + +app_name = 'dpp' + +urlpatterns = [ + path("/", views.LotDashboardView.as_view(), name="proof"), +] diff --git a/dpp/views.py b/dpp/views.py index 91ea44a..956099d 100644 --- a/dpp/views.py +++ b/dpp/views.py @@ -1,3 +1,36 @@ -from django.shortcuts import render +from django.views.generic.edit import View +from django.http import JsonResponse -# Create your views here. +from evidence.xapian import search +from dpp.models import Proof +from dpp.api_dlt import ALGORITHM + + +class ProofView(View): + + def get(self, request, *args, **kwargs): + timestamp = kwargs.get("proof_id") + proof = Proof.objects.filter(timestamp=timestamp).first() + if not proof: + return JsonResponse({}, status=404) + + ev_uuid = 'uuid:"{}"'.format(proof.uuid) + matches = search(None, ev_uuid, limit=1) + if not matches or matches.size() < 1: + return JsonResponse({}, status=404) + + for x in matches: + snap = x.document.get_data() + + data = { + "algorithm": ALGORITHM, + "document": snap + } + + d = { + '@context': ['https://ereuse.org/proof0.json'], + 'data': data, + } + return JsonResponse(d, status=200) + + return JsonResponse({}, status=404) diff --git a/evidence/models.py b/evidence/models.py index e9af092..f85fd58 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -1,4 +1,5 @@ import json +import hashlib from dmidecode import DMIParse from django.db import models @@ -58,6 +59,12 @@ class Evidence: if a: self.owner = a.owner + def get_phid(self): + if not self.doc: + self.get_doc() + + return hashlib.sha3_256(json.dumps(self.doc)).hexdigest() + def get_doc(self): self.doc = {} if not self.owner: diff --git a/evidence/parse.py b/evidence/parse.py index fd68e06..b912d50 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -8,11 +8,13 @@ from evidence.parse_details import get_lshw_child from evidence.models import Annotation from evidence.xapian import index +from dpp.api_dlt import register_device_dlt, register_passport_dlt from utils.constants import CHASSIS_DH logger = logging.getLogger('django') + def get_mac(lshw): try: if type(lshw) is dict: @@ -40,6 +42,8 @@ class Build: self.uuid = self.json['uuid'] self.user = user self.hid = None + self.chid = None + self.phid = self.get_signature(self.json) self.generate_chids() if check: @@ -47,6 +51,7 @@ class Build: self.index() self.create_annotations() + self.register_device_dlt() def index(self): snap = json.dumps(self.json) @@ -70,7 +75,8 @@ class Build: hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}" - return hashlib.sha3_256(hid.encode()).hexdigest() + self.chid = hashlib.sha3_256(hid.encode()).hexdigest() + return self.chid def create_annotations(self): annotation = Annotation.objects.filter( @@ -129,3 +135,10 @@ class Build: logger.warning(txt, snapshot['uuid']) return f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}" + + def get_signature(self, doc): + return hashlib.sha3_256(json.dumps(doc).encode()).hexdigest() + + def register_device_dlt(self): + register_device_dlt(self.chid, self.phid, self.uuid, self.user) + register_passport_dlt(self.chid, self.phid, self.uuid, self.user) diff --git a/evidence/xapian.py b/evidence/xapian.py index 98da706..2e1cd9f 100644 --- a/evidence/xapian.py +++ b/evidence/xapian.py @@ -22,10 +22,14 @@ def search(institution, qs, offset=0, limit=10): qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME) qp.add_prefix("uuid", "uuid") query = qp.parse_query(qs) - institution_term = "U{}".format(institution.id) - final_query = xapian.Query( - xapian.Query.OP_AND, query, xapian.Query(institution_term) - ) + if institution: + institution_term = "U{}".format(institution.id) + final_query = xapian.Query( + xapian.Query.OP_AND, query, xapian.Query(institution_term) + ) + else: + final_query = xapian.Query(query) + enquire = xapian.Enquire(database) enquire.set_query(final_query) matches = enquire.get_mset(offset, limit) From cc350775ed7be2b63a10030d617fdbc043ecbe22 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 19 Nov 2024 18:02:43 +0100 Subject: [PATCH 03/80] . --- device/templates/details.html | 4 ++-- device/views.py | 2 +- dhub/settings.py | 5 +++++ dpp/api_dlt.py | 24 +++++++++++++++++------- dpp/urls.py | 2 +- requirements.txt | 3 +++ 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/device/templates/details.html b/device/templates/details.html index 9783057..0670988 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -30,7 +30,7 @@ {% trans 'Evidences' %} + {% if dpps %} + {% endif %} @@ -232,7 +234,8 @@ {% endfor %} - + + {% if dpps %}
{% trans 'List of dpps' %}
@@ -249,6 +252,7 @@ {% endfor %}
+ {% endif %} {% endblock %} @@ -257,12 +261,12 @@ document.addEventListener('DOMContentLoaded', function () { // Obtener el hash de la URL (ejemplo: #components) const hash = window.location.hash - + // Verificar si hay un hash en la URL if (hash) { // Buscar el botón o enlace que corresponde al hash y activarlo const tabTrigger = document.querySelector(`[data-bs-target="${hash}"]`) - + if (tabTrigger) { // Crear una instancia de tab de Bootstrap para activar el tab const tab = new bootstrap.Tab(tabTrigger) diff --git a/device/views.py b/device/views.py index c7c2b1f..3baa054 100644 --- a/device/views.py +++ b/device/views.py @@ -1,6 +1,5 @@ from django.http import JsonResponse - -from django.http import Http404 +from django.conf import settings from django.urls import reverse_lazy from django.shortcuts import get_object_or_404, Http404 from django.utils.translation import gettext_lazy as _ @@ -13,10 +12,11 @@ from django.views.generic.base import TemplateView from dashboard.mixins import DashboardView, Http403 from evidence.models import Annotation from lot.models import LotTag -from dpp.models import Proof -from dpp.api_dlt import PROOF_TYPE from device.models import Device from device.forms import DeviceFormSet +if settings.DPP: + from dpp.models import Proof + from dpp.api_dlt import PROOF_TYPE class NewDeviceView(DashboardView, FormView): @@ -104,10 +104,12 @@ class DetailsView(DashboardView, TemplateView): context = super().get_context_data(**kwargs) self.object.initial() lot_tags = LotTag.objects.filter(owner=self.request.user.institution) - dpps = Proof.objects.filter( - uuid__in=self.object.uuids, - type=PROOF_TYPE["IssueDPP"] - ) + dpps = [] + if settings.DPP: + dpps = Proof.objects.filter( + uuid__in=self.object.uuids, + type=PROOF_TYPE["IssueDPP"] + ) context.update({ 'object': self.object, 'snapshot': self.object.get_last_evidence(), diff --git a/dhub/settings.py b/dhub/settings.py index a30ab2b..2aca54e 100644 --- a/dhub/settings.py +++ b/dhub/settings.py @@ -89,10 +89,13 @@ INSTALLED_APPS = [ "dashboard", "admin", "api", - "dpp", - "did", ] +DPP = config("DPP", default=False, cast=bool) + +if DPP: + INSTALLED_APPS.extend(["dpp", "did"]) + MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", diff --git a/dhub/urls.py b/dhub/urls.py index d070a15..0b77144 100644 --- a/dhub/urls.py +++ b/dhub/urls.py @@ -14,7 +14,7 @@ Including another URLconf 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ - +from django.conf import settings from django.urls import path, include urlpatterns = [ @@ -27,6 +27,10 @@ urlpatterns = [ path("user/", include("user.urls")), path("lot/", include("lot.urls")), path('api/', include('api.urls')), - path('dpp/', include('dpp.urls')), - path('did/', include('did.urls')), ] + +if settings.DPP: + urlpatterns.extend([ + path('dpp/', include('dpp.urls')), + path('did/', include('did.urls')), + ]) diff --git a/evidence/parse.py b/evidence/parse.py index 6a78285..3752bf5 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -4,12 +4,14 @@ import logging from dmidecode import DMIParse from json_repair import repair_json +from django.conf import settings from evidence.parse_details import get_lshw_child, ParseSnapshot from evidence.models import Annotation from evidence.xapian import index -from dpp.api_dlt import register_device_dlt, register_passport_dlt from utils.constants import CHASSIS_DH +if settings.DPP: + from dpp.api_dlt import register_device_dlt, register_passport_dlt logger = logging.getLogger('django') @@ -51,7 +53,8 @@ class Build: self.index() self.create_annotations() - self.register_device_dlt() + if settings.DPP: + self.register_device_dlt() def index(self): snap = json.dumps(self.json) diff --git a/evidence/views.py b/evidence/views.py index dabcd9d..c2868b2 100644 --- a/evidence/views.py +++ b/evidence/views.py @@ -137,7 +137,7 @@ class DownloadEvidenceView(DashboardView, TemplateView): evidence.get_doc() data = json.dumps(evidence.doc) response = HttpResponse(data, content_type="application/json") - response['Content-Disposition'] = 'attachment; filename={}'.format("credential.json") + response['Content-Disposition'] = 'attachment; filename={}'.format("evidence.json") return response From 3bf23c5d3b6ccb742a9c42d9215eb4d84709015f Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 11 Dec 2024 11:16:32 +0100 Subject: [PATCH 66/80] propagate DPP env var to docker --- .env.example | 1 + docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.env.example b/.env.example index ed60e95..d8fed15 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ DEMO=true # note that with DEBUG=true, logs are more verbose (include tracebacks) DEBUG=true ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1, +DPP=false STATIC_ROOT=/tmp/static/ MEDIA_ROOT=/tmp/media/ diff --git a/docker-compose.yml b/docker-compose.yml index 9b82d7d..9a156c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - ALLOWED_HOSTS=${ALLOWED_HOSTS:-$DOMAIN} - DEMO=${DEMO:-false} - PREDEFINED_TOKEN=${PREDEFINED_TOKEN:-} + - DPP=${DPP:-false} volumes: - .:/opt/devicehub-django ports: From c5dd3a759dce5cc0579416159fd1925aeb13ce4e Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 11 Dec 2024 11:23:35 +0100 Subject: [PATCH 67/80] docker entrypoint: adapt it to DPP env var --- docker/devicehub-django.entrypoint.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index de19a4d..73a0f19 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -34,7 +34,7 @@ gen_env_vars() { ADMIN='True' PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}" # specific dpp env vars - if [ "${DPP_MODULE}" = 'y' ]; then + if [ "${DPP}" = 'true' ]; then # fill env vars in this docker entrypoint wait_for_dpp_shared export API_DLT='http://api_connector:3010' @@ -129,11 +129,12 @@ config_phase() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DPP_MODULE}" = 'y' ]; then + if [ "${DPP}" = 'true' ]; then # 12, 13, 14 config_dpp_part1 # cleanup other spnapshots and copy dlt/dpp snapshots + # TODO make this better rm example/snapshots/* cp example/dpp-snapshots/*.json example/snapshots/ fi From f5ea0f8d8d84d7d3697da1182db9f1862a22f069 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 11 Dec 2024 11:25:35 +0100 Subject: [PATCH 68/80] docker entrypoint: bugfix when DPP env var unbound --- docker/devicehub-django.entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index 73a0f19..d6d7bb2 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -34,7 +34,7 @@ gen_env_vars() { ADMIN='True' PREDEFINED_TOKEN="${PREDEFINED_TOKEN:-}" # specific dpp env vars - if [ "${DPP}" = 'true' ]; then + if [ "${DPP:-}" = 'true' ]; then # fill env vars in this docker entrypoint wait_for_dpp_shared export API_DLT='http://api_connector:3010' @@ -129,7 +129,7 @@ config_phase() { # TODO: one error on add_user, and you don't add user anymore ./manage.py add_user "${INIT_ORG}" "${INIT_USER}" "${INIT_PASSWD}" "${ADMIN}" "${PREDEFINED_TOKEN}" - if [ "${DPP}" = 'true' ]; then + if [ "${DPP:-}" = 'true' ]; then # 12, 13, 14 config_dpp_part1 From dbdcffe5d98b497164a448a7f6ea132a4a1c47ec Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 15 Nov 2024 12:47:08 +0100 Subject: [PATCH 69/80] add inxi in parsing and show in details of devs --- dashboard/templates/unassigned_devices.html | 4 + device/models.py | 6 + device/templates/details.html | 17 +- evidence/models.py | 38 +- evidence/parse.py | 58 +- evidence/parse_details.py | 619 ++++++++------------ 6 files changed, 342 insertions(+), 400 deletions(-) diff --git a/dashboard/templates/unassigned_devices.html b/dashboard/templates/unassigned_devices.html index 369edd2..d54e450 100644 --- a/dashboard/templates/unassigned_devices.html +++ b/dashboard/templates/unassigned_devices.html @@ -69,7 +69,11 @@ {{ dev.manufacturer }} + {% if dev.version %} + {{dev.version}} {{ dev.model }} + {% else %} {{ dev.model }} + {% endif %} diff --git a/device/models.py b/device/models.py index cd77389..7ca2a1f 100644 --- a/device/models.py +++ b/device/models.py @@ -305,6 +305,12 @@ class Device: self.get_last_evidence() return self.last_evidence.get_model() + @property + def version(self): + if not self.last_evidence: + self.get_last_evidence() + return self.last_evidence.get_version() + @property def components(self): self.get_last_evidence() diff --git a/device/templates/details.html b/device/templates/details.html index 99346c9..af6ce7c 100644 --- a/device/templates/details.html +++ b/device/templates/details.html @@ -58,34 +58,41 @@ {% endif %} -
+
Type
{{ object.type }}
{% if object.is_websnapshot and object.last_user_evidence %} {% for k, v in object.last_user_evidence %} -
+
{{ k }}
{{ v|default:'' }}
{% endfor %} {% else %} -
+
{% trans 'Manufacturer' %}
{{ object.manufacturer|default:'' }}
-
+
{% trans 'Model' %}
{{ object.model|default:'' }}
-
+
+
+ {% trans 'Version' %} +
+
{{ object.version|default:'' }}
+
+ +
{% trans 'Serial Number' %}
diff --git a/evidence/models.py b/evidence/models.py index f85fd58..99e6a96 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -6,7 +6,7 @@ from django.db import models from utils.constants import STR_EXTEND_SIZE, CHASSIS_DH from evidence.xapian import search -from evidence.parse_details import ParseSnapshot +from evidence.parse_details import ParseSnapshot, get_inxi, get_inxi_key from user.models import User, Institution @@ -40,6 +40,7 @@ class Evidence: self.doc = None self.created = None self.dmi = None + self.inxi = None self.annotations = [] self.components = [] self.default = "n/a" @@ -79,7 +80,22 @@ class Evidence: if not self.is_legacy(): dmidecode_raw = self.doc["data"]["dmidecode"] + inxi_raw = self.doc["data"]["inxi"] self.dmi = DMIParse(dmidecode_raw) + try: + self.inxi = json.loads(inxi_raw) + machine = get_inxi_key(self.inxi, 'Machine') + for m in machine: + system = get_inxi(m, "System") + if system: + self.device_manufacturer = system + self.device_model = get_inxi(m, "product") + self.device_serial_number = get_inxi(m, "serial") + self.device_chassis = get_inxi(m, "Type") + self.device_version = get_inxi(m, "v") + + except Exception: + return def get_time(self): if not self.doc: @@ -105,6 +121,9 @@ class Evidence: if self.is_legacy(): return self.doc['device']['manufacturer'] + if self.inxi: + return self.device_manufacturer + return self.dmi.manufacturer().strip() def get_model(self): @@ -117,11 +136,17 @@ class Evidence: if self.is_legacy(): return self.doc['device']['model'] + if self.inxi: + return self.device_model + return self.dmi.model().strip() def get_chassis(self): if self.is_legacy(): return self.doc['device']['model'] + + if self.inxi: + return self.device_chassis chassis = self.dmi.get("Chassis")[0].get("Type", '_virtual') lower_type = chassis.lower() @@ -134,8 +159,19 @@ class Evidence: def get_serial_number(self): if self.is_legacy(): return self.doc['device']['serialNumber'] + + if self.inxi: + return self.device_serial_number + return self.dmi.serial_number().strip() + + def get_version(self): + if self.inxi: + return self.device_version + + return "" + @classmethod def get_all(cls, user): return Annotation.objects.filter( diff --git a/evidence/parse.py b/evidence/parse.py index 3752bf5..9dca596 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -10,32 +10,24 @@ from evidence.parse_details import get_lshw_child, ParseSnapshot from evidence.models import Annotation from evidence.xapian import index from utils.constants import CHASSIS_DH + if settings.DPP: from dpp.api_dlt import register_device_dlt, register_passport_dlt + +from evidence.parse_details import get_inxi_key, get_inxi + logger = logging.getLogger('django') -def get_mac(lshw): - try: - if type(lshw) is dict: - hw = lshw - else: - hw = json.loads(lshw) - except json.decoder.JSONDecodeError: - hw = json.loads(repair_json(lshw)) - - nets = [] - get_lshw_child(hw, nets, 'network') - - nets_sorted = sorted(nets, key=lambda x: x['businfo']) - - if nets_sorted: - mac = nets_sorted[0]['serial'] - logger.debug("The snapshot has the following MAC: %s" , mac) - return mac +def get_mac(inxi): + nets = get_inxi_key(inxi, "Network") + networks = [(nets[i], nets[i + 1]) for i in range(0, len(nets) - 1, 2)] + for n, iface in networks: + if get_inxi(n, "port"): + return get_inxi(iface, 'mac') class Build: @@ -175,24 +167,28 @@ class Build: return self.dmi.get("System")[0].get("Verson", '_virtual') def get_hid(self, snapshot): - dmidecode_raw = snapshot["data"]["dmidecode"] - self.dmi = DMIParse(dmidecode_raw) + try: + self.inxi = json.loads(self.json["inxi"]) + except Exception: + logger.error("No inxi in snapshot %s", self.uuid) + return "" - manufacturer = self.dmi.manufacturer().strip() - model = self.dmi.model().strip() - chassis = self.get_chassis_dh() - serial_number = self.dmi.serial_number() - sku = self.get_sku() + machine = get_inxi_key(self.inxi, 'Machine') + for m in machine: + system = get_inxi(m, "System") + if system: + manufacturer = system + model = get_inxi(m, "product") + serial_number = get_inxi(m, "serial") + chassis = get_inxi(m, "Type") + else: + sku = get_inxi(m, "part-nu") - if not snapshot["data"].get('lshw'): - return f"{manufacturer}{model}{chassis}{serial_number}{sku}" - - lshw = snapshot["data"]["lshw"] - # mac = get_mac2(hwinfo_raw) or "" - mac = get_mac(lshw) or "" + mac = get_mac(self.inxi) or "" if not mac: txt = "Could not retrieve MAC address in snapshot %s" logger.warning(txt, snapshot['uuid']) + return f"{manufacturer}{model}{chassis}{serial_number}{sku}" return f"{manufacturer}{model}{chassis}{serial_number}{sku}{mac}" diff --git a/evidence/parse_details.py b/evidence/parse_details.py index 12ade8a..55ecdd7 100644 --- a/evidence/parse_details.py +++ b/evidence/parse_details.py @@ -1,10 +1,10 @@ +import re import json import logging import numpy as np from datetime import datetime from dmidecode import DMIParse -from json_repair import repair_json from utils.constants import CHASSIS_DH, DATASTORAGEINTERFACE @@ -12,11 +12,19 @@ from utils.constants import CHASSIS_DH, DATASTORAGEINTERFACE logger = logging.getLogger('django') -def get_lshw_child(child, nets, component): - if child.get('id') == component: - nets.append(child) - if child.get('children'): - [get_lshw_child(x, nets, component) for x in child['children']] +def get_inxi_key(inxi, component): + for n in inxi: + for k, v in n.items(): + if component in k: + return v + + +def get_inxi(n, name): + for k, v in n.items(): + if f"#{name}" in k: + return v + + return "" class ParseSnapshot: @@ -24,20 +32,15 @@ class ParseSnapshot: self.default = default self.dmidecode_raw = snapshot["data"].get("dmidecode", "{}") self.smart_raw = snapshot["data"].get("disks", []) - self.hwinfo_raw = snapshot["data"].get("hwinfo", "") - self.lshw_raw = snapshot["data"].get("lshw", {}) or {} - self.lscpi_raw = snapshot["data"].get("lspci", "") + self.inxi_raw = snapshot["data"].get("inxi", "") or "" self.device = {"actions": []} self.components = [] - self.monitors = [] self.dmi = DMIParse(self.dmidecode_raw) self.smart = self.loads(self.smart_raw) - self.lshw = self.loads(self.lshw_raw) - self.hwinfo = self.parse_hwinfo() + self.inxi = self.loads(self.inxi_raw) self.set_computer() - self.get_hwinfo_monitors() self.set_components() self.snapshot_json = { "type": "Snapshot", @@ -51,283 +54,294 @@ class ParseSnapshot: } def set_computer(self): - self.device['manufacturer'] = self.dmi.manufacturer().strip() - self.device['model'] = self.dmi.model().strip() - self.device['serialNumber'] = self.dmi.serial_number() - self.device['type'] = self.get_type() - self.device['sku'] = self.get_sku() - self.device['version'] = self.get_version() - self.device['system_uuid'] = self.get_uuid() - self.device['family'] = self.get_family() - self.device['chassis'] = self.get_chassis_dh() + machine = get_inxi_key(self.inxi, 'Machine') + for m in machine: + system = get_inxi(m, "System") + if system: + self.device['manufacturer'] = system + self.device['model'] = get_inxi(m, "product") + self.device['serialNumber'] = get_inxi(m, "serial") + self.device['type'] = get_inxi(m, "Type") + self.device['chassis'] = self.device['type'] + self.device['version'] = get_inxi(m, "v") + else: + self.device['system_uuid'] = get_inxi(m, "uuid") + self.device['sku'] = get_inxi(m, "part-nu") def set_components(self): + self.get_mother_board() self.get_cpu() self.get_ram() - self.get_mother_board() self.get_graphic() - self.get_data_storage() self.get_display() - self.get_sound_card() self.get_networks() - - def get_cpu(self): - for cpu in self.dmi.get('Processor'): - serial = cpu.get('Serial Number') - if serial == 'Not Specified' or not serial: - serial = cpu.get('ID').replace(' ', '') - self.components.append( - { - "actions": [], - "type": "Processor", - "speed": self.get_cpu_speed(cpu), - "cores": int(cpu.get('Core Count', 1)), - "model": cpu.get('Version'), - "threads": int(cpu.get('Thread Count', 1)), - "manufacturer": cpu.get('Manufacturer'), - "serialNumber": serial, - "brand": cpu.get('Family'), - "address": self.get_cpu_address(cpu), - "bogomips": self.get_bogomips(), - } - ) - - def get_ram(self): - for ram in self.dmi.get("Memory Device"): - if ram.get('size') == 'No Module Installed': - continue - if not ram.get("Speed"): - continue - - self.components.append( - { - "actions": [], - "type": "RamModule", - "size": self.get_ram_size(ram), - "speed": self.get_ram_speed(ram), - "manufacturer": ram.get("Manufacturer", self.default), - "serialNumber": ram.get("Serial Number", self.default), - "interface": ram.get("Type", "DDR"), - "format": ram.get("Form Factor", "DIMM"), - "model": ram.get("Part Number", self.default), - } - ) + self.get_sound_card() + self.get_data_storage() + self.get_battery() def get_mother_board(self): - for moder_board in self.dmi.get("Baseboard"): - self.components.append( - { - "actions": [], - "type": "Motherboard", - "version": moder_board.get("Version"), - "serialNumber": moder_board.get("Serial Number", "").strip(), - "manufacturer": moder_board.get("Manufacturer", "").strip(), - "biosDate": self.get_bios_date(), - "ramMaxSize": self.get_max_ram_size(), - "ramSlots": len(self.dmi.get("Memory Device")), - "slots": self.get_ram_slots(), - "model": moder_board.get("Product Name", "").strip(), - "firewire": self.get_firmware_num(), - "pcmcia": self.get_pcmcia_num(), - "serial": self.get_serial_num(), - "usb": self.get_usb_num(), - } - ) + machine = get_inxi_key(self.inxi, 'Machine') + mb = {"type": "Motherboard",} + for m in machine: + bios_date = get_inxi(m, "date") + if not bios_date: + continue + mb["manufacturer"] = get_inxi(m, "Mobo") + mb["model"] = get_inxi(m, "model") + mb["serialNumber"] = get_inxi(m, "serial") + mb["version"] = get_inxi(m, "v") + mb["biosDate"] = bios_date + mb["biosVersion"] = self.get_bios_version() + mb["firewire"]: self.get_firmware_num() + mb["pcmcia"]: self.get_pcmcia_num() + mb["serial"]: self.get_serial_num() + mb["usb"]: self.get_usb_num() + + self.get_ram_slots(mb) + + self.components.append(mb) + + def get_ram_slots(self, mb): + memory = get_inxi_key(self.inxi, 'Memory') + for m in memory: + slots = get_inxi(m, "slots") + if not slots: + continue + mb["slots"] = slots + mb["ramSlots"] = get_inxi(m, "modules") + mb["ramMaxSize"] = get_inxi(m, "capacity") + + + def get_cpu(self): + cpu = get_inxi_key(self.inxi, 'CPU') + cp = {"type": "Processor"} + vulnerabilities = [] + for c in cpu: + base = get_inxi(c, "model") + if base: + cp["model"] = get_inxi(c, "model") + cp["arch"] = get_inxi(c, "arch") + cp["bits"] = get_inxi(c, "bits") + cp["gen"] = get_inxi(c, "gen") + cp["family"] = get_inxi(c, "family") + cp["date"] = get_inxi(c, "built") + continue + des = get_inxi(c, "L1") + if des: + cp["L1"] = des + cp["L2"] = get_inxi(c, "L2") + cp["L3"] = get_inxi(c, "L3") + cp["cpus"] = get_inxi(c, "cpus") + cp["cores"] = get_inxi(c, "cores") + cp["threads"] = get_inxi(c, "threads") + continue + bogo = get_inxi(c, "bogomips") + if bogo: + cp["bogomips"] = bogo + cp["base/boost"] = get_inxi(c, "base/boost") + cp["min/max"] = get_inxi(c, "min/max") + cp["ext-clock"] = get_inxi(c, "ext-clock") + cp["volts"] = get_inxi(c, "volts") + continue + ctype = get_inxi(c, "Type") + if ctype: + v = {"Type": ctype} + status = get_inxi(c, "status") + if status: + v["status"] = status + mitigation = get_inxi(c, "mitigation") + if mitigation: + v["mitigation"] = mitigation + vulnerabilities.append(v) + + self.components.append(cp) + + + def get_ram(self): + memory = get_inxi_key(self.inxi, 'Memory') + mem = {"type": "RamModule"} + + for m in memory: + base = get_inxi(m, "System RAM") + if base: + mem["size"] = get_inxi(m, "total") + slot = get_inxi(m, "manufacturer") + if slot: + mem["manufacturer"] = slot + mem["model"] = get_inxi(m, "part-no") + mem["serialNumber"] = get_inxi(m, "serial") + mem["speed"] = get_inxi(m, "speed") + mem["bits"] = get_inxi(m, "data") + mem["interface"] = get_inxi(m, "type") + module = get_inxi(m, "modules") + if module: + mem["modules"] = module + + self.components.append(mem) def get_graphic(self): - displays = [] - get_lshw_child(self.lshw, displays, 'display') - - for c in displays: - if not c['configuration'].get('driver', None): + graphics = get_inxi_key(self.inxi, 'Graphics') + + for c in graphics: + if not get_inxi(c, "Device") or not get_inxi(c, "vendor"): continue self.components.append( { - "actions": [], "type": "GraphicCard", "memory": self.get_memory_video(c), - "manufacturer": c.get("vendor", self.default), - "model": c.get("product", self.default), - "serialNumber": c.get("serial", self.default), + "manufacturer": get_inxi(c, "vendor"), + "model": get_inxi(c, "Device"), + "arch": get_inxi(c, "arch"), + "serialNumber": get_inxi(c, "serial"), + "integrated": True if get_inxi(c, "port") else False + } + ) + + def get_battery(self): + bats = get_inxi_key(self.inxi, 'Battery') + for b in bats: + self.components.append( + { + "type": "Battery", + "model": get_inxi(b, "model"), + "serialNumber": get_inxi(b, "serial"), + "condition": get_inxi(b, "condition"), + "cycles": get_inxi(b, "cycles"), + "volts": get_inxi(b, "volts") } ) def get_memory_video(self, c): - # get info of lspci - # pci_id = c['businfo'].split('@')[1] - # lspci.get(pci_id) | grep size - # lspci -v -s 00:02.0 - return None + memory = get_inxi_key(self.inxi, 'Memory') + + for m in memory: + igpu = get_inxi(m, "igpu") + agpu = get_inxi(m, "agpu") + ngpu = get_inxi(m, "ngpu") + gpu = get_inxi(m, "gpu") + if igpu or agpu or gpu or ngpu: + return igpu or agpu or gpu or ngpu + + return self.default def get_data_storage(self): - for sm in self.smart: - if sm.get('smartctl', {}).get('exit_status') == 1: + hdds= get_inxi_key(self.inxi, 'Drives') + for d in hdds: + usb = get_inxi(d, "type") + if usb == "USB": continue - model = sm.get('model_name') - manufacturer = None - hours = sm.get("power_on_time", {}).get("hours", 0) - if model and len(model.split(" ")) > 1: - mm = model.split(" ") - model = mm[-1] - manufacturer = " ".join(mm[:-1]) - self.components.append( - { - "actions": self.sanitize(sm), - "type": self.get_data_storage_type(sm), - "model": model, - "manufacturer": manufacturer, - "serialNumber": sm.get('serial_number'), - "size": self.get_data_storage_size(sm), - "variant": sm.get("firmware_version"), - "interface": self.get_data_storage_interface(sm), - "hours": hours, + serial = get_inxi(d, "serial") + if serial: + hd = { + "type": "Storage", + "manufacturer": get_inxi(d, "vendor"), + "model": get_inxi(d, "model"), + "serialNumber": get_inxi(d, "serial"), + "size": get_inxi(d, "size"), + "speed": get_inxi(d, "speed"), + "interface": get_inxi(d, "tech"), + "firmware": get_inxi(d, "fw-rev") } - ) + rpm = get_inxi(d, "rpm") + if rpm: + hd["rpm"] = rpm + + family = get_inxi(d, "family") + if family: + hd["family"] = family + + sata = get_inxi(d, "sata") + if sata: + hd["sata"] = sata + + continue + + + cycles = get_inxi(d, "cycles") + if cycles: + hd['cycles'] = cycles + hd["health"] = get_inxi(d, "health") + hd["time of used"] = get_inxi(d, "on") + hd["read used"] = get_inxi(d, "read-units") + hd["written used"] = get_inxi(d, "written-units") + + # import pdb; pdb.set_trace() + self.components.append(hd) + continue + + hd = {} def sanitize(self, action): return [] - def get_bogomips(self): - if not self.hwinfo: - return self.default - - bogomips = 0 - for row in self.hwinfo: - for cel in row: - if 'BogoMips' in cel: - try: - bogomips += float(cel.split(":")[-1]) - except: - pass - return bogomips - def get_networks(self): - networks = [] - get_lshw_child(self.lshw, networks, 'network') - - for c in networks: - capacity = c.get('capacity') - wireless = bool(c.get('configuration', {}).get('wireless', False)) + nets = get_inxi_key(self.inxi, "Network") + networks = [(nets[i], nets[i + 1]) for i in range(0, len(nets) - 1, 2)] + + for n, iface in networks: + model = get_inxi(n, "Device") + if not model: + continue + + interface = '' + for k in n.keys(): + if "port" in k: + interface = "Integrated" + if "pcie" in k: + interface = "PciExpress" + if get_inxi(n, "type") == "USB": + interface = "USB" + self.components.append( { - "actions": [], "type": "NetworkAdapter", - "model": c.get('product'), - "manufacturer": c.get('vendor'), - "serialNumber": c.get('serial'), - "speed": capacity, - "variant": c.get('version', 1), - "wireless": wireless or False, - "integrated": "PCI:0000:00" in c.get("businfo", ""), + "model": model, + "manufacturer": get_inxi(n, 'vendor'), + "serialNumber": get_inxi(iface, 'mac'), + "speed": get_inxi(n, "speed"), + "interface": interface, } ) def get_sound_card(self): - multimedias = [] - get_lshw_child(self.lshw, multimedias, 'multimedia') - - for c in multimedias: + audio = get_inxi_key(self.inxi, "Audio") + + for c in audio: + model = get_inxi(c, "Device") + if not model: + continue + self.components.append( { - "actions": [], "type": "SoundCard", - "model": c.get('product'), - "manufacturer": c.get('vendor'), - "serialNumber": c.get('serial'), + "model": model, + "manufacturer": get_inxi(c, 'vendor'), + "serialNumber": get_inxi(c, 'serial'), } ) - def get_display(self): # noqa: C901 - TECHS = 'CRT', 'TFT', 'LED', 'PDP', 'LCD', 'OLED', 'AMOLED' - - for c in self.monitors: - resolution_width, resolution_height = (None,) * 2 - refresh, serial, model, manufacturer, size = (None,) * 5 - year, week, production_date = (None,) * 3 - - for x in c: - if "Vendor: " in x: - manufacturer = x.split('Vendor: ')[-1].strip() - if "Model: " in x: - model = x.split('Model: ')[-1].strip() - if "Serial ID: " in x: - serial = x.split('Serial ID: ')[-1].strip() - if " Resolution: " in x: - rs = x.split(' Resolution: ')[-1].strip() - if 'x' in rs: - resolution_width, resolution_height = [ - int(r) for r in rs.split('x') - ] - if "Frequencies: " in x: - try: - refresh = int(float(x.split(',')[-1].strip()[:-3])) - except Exception: - pass - if 'Year of Manufacture' in x: - year = x.split(': ')[1] - - if 'Week of Manufacture' in x: - week = x.split(': ')[1] - - if "Size: " in x: - size = self.get_size_monitor(x) - technology = next((t for t in TECHS if t in c[0]), None) - - if year and week: - d = '{} {} 0'.format(year, week) - production_date = datetime.strptime(d, '%Y %W %w').isoformat() + def get_display(self): + graphics = get_inxi_key(self.inxi, "Graphics") + for c in graphics: + if not get_inxi(c, "Monitor"): + continue self.components.append( { - "actions": [], "type": "Display", - "model": model, - "manufacturer": manufacturer, - "serialNumber": serial, - 'size': size, - 'resolutionWidth': resolution_width, - 'resolutionHeight': resolution_height, - "productionDate": production_date, - 'technology': technology, - 'refreshRate': refresh, + "model": get_inxi(c, "model"), + "manufacturer": get_inxi(c, "vendor"), + "serialNumber": get_inxi(c, "serial"), + 'size': get_inxi(c, "size"), + 'diagonal': get_inxi(c, "diag"), + 'resolution': get_inxi(c, "res"), + "date": get_inxi(c, "built"), + 'ratio': get_inxi(c, "ratio"), } ) - def get_hwinfo_monitors(self): - for c in self.hwinfo: - monitor = None - external = None - for x in c: - if 'Hardware Class: monitor' in x: - monitor = c - if 'Driver Info' in x: - external = c - - if monitor and not external: - self.monitors.append(c) - - def get_size_monitor(self, x): - i = 1 / 25.4 - t = x.split('Size: ')[-1].strip() - tt = t.split('mm') - if not tt: - return 0 - sizes = tt[0].strip() - if 'x' not in sizes: - return 0 - w, h = [int(x) for x in sizes.split('x')] - return "{:.2f}".format(np.sqrt(w**2 + h**2) * i) - - def get_cpu_address(self, cpu): - default = 64 - for ch in self.lshw.get('children', []): - for c in ch.get('children', []): - if c['class'] == 'processor': - return c.get('width', default) - return default - def get_usb_num(self): return len( [ @@ -364,133 +378,13 @@ class ParseSnapshot: ] ) - def get_bios_date(self): - return self.dmi.get("BIOS")[0].get("Release Date", self.default) - - def get_firmware(self): - return self.dmi.get("BIOS")[0].get("Firmware Revision", '1') - - def get_max_ram_size(self): - size = 0 - for slot in self.dmi.get("Physical Memory Array"): - capacity = slot.get("Maximum Capacity", '0').split(" ")[0] - size += int(capacity) - - return size - - def get_ram_slots(self): - slots = 0 - for x in self.dmi.get("Physical Memory Array"): - slots += int(x.get("Number Of Devices", 0)) - return slots - - def get_ram_size(self, ram): - memory = ram.get("Size", "0") - return memory - - def get_ram_speed(self, ram): - size = ram.get("Speed", "0") - return size - - def get_cpu_speed(self, cpu): - speed = cpu.get('Max Speed', "0") - return speed - - def get_sku(self): - return self.dmi.get("System")[0].get("SKU Number", self.default).strip() - - def get_version(self): - return self.dmi.get("System")[0].get("Version", self.default).strip() - - def get_uuid(self): - return self.dmi.get("System")[0].get("UUID", '').strip() - - def get_family(self): - return self.dmi.get("System")[0].get("Family", '') - - def get_chassis(self): - return self.dmi.get("Chassis")[0].get("Type", '_virtual') - - def get_type(self): - chassis_type = self.get_chassis() - return self.translation_to_devicehub(chassis_type) - - def translation_to_devicehub(self, original_type): - lower_type = original_type.lower() - CHASSIS_TYPE = { - 'Desktop': [ - 'desktop', - 'low-profile', - 'tower', - 'docking', - 'all-in-one', - 'pizzabox', - 'mini-tower', - 'space-saving', - 'lunchbox', - 'mini', - 'stick', - ], - 'Laptop': [ - 'portable', - 'laptop', - 'convertible', - 'tablet', - 'detachable', - 'notebook', - 'handheld', - 'sub-notebook', - ], - 'Server': ['server'], - 'Computer': ['_virtual'], - } - for k, v in CHASSIS_TYPE.items(): - if lower_type in v: - return k - return self.default - - def get_chassis_dh(self): - chassis = self.get_chassis() - lower_type = chassis.lower() - for k, v in CHASSIS_DH.items(): - if lower_type in v: - return k - return self.default - - def get_data_storage_type(self, x): - # TODO @cayop add more SSDS types - SSDS = ["nvme"] - SSD = 'SolidStateDrive' - HDD = 'HardDrive' - type_dev = x.get('device', {}).get('type') - trim = x.get('trim', {}).get("supported") in [True, "true"] - return SSD if type_dev in SSDS or trim else HDD - - def get_data_storage_interface(self, x): - interface = x.get('device', {}).get('protocol', 'ATA') - if interface.upper() in DATASTORAGEINTERFACE: - return interface.upper() - - txt = "Sid: {}, interface {} is not in DataStorageInterface Enum".format( - self.sid, interface - ) - self.errors("{}".format(err)) - - def get_data_storage_size(self, x): - return x.get('user_capacity', {}).get('bytes') - - def parse_hwinfo(self): - hw_blocks = self.hwinfo_raw.split("\n\n") - return [x.split("\n") for x in hw_blocks] + def get_bios_version(self): + return self.dmi.get("BIOS")[0].get("BIOS Revision", '1') def loads(self, x): if isinstance(x, str): try: - try: - hw = json.loads(x) - except json.decoder.JSONDecodeError: - hw = json.loads(repair_json(x)) - return hw + return json.loads(x) except Exception as ss: logger.warning("%s", ss) return {} @@ -502,4 +396,3 @@ class ParseSnapshot: logger.error(txt) self._errors.append("%s", txt) - From 989120bd0aaaf684b5d2a79730532c6abc186dce Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 20 Nov 2024 18:35:27 +0100 Subject: [PATCH 70/80] fix component empty --- evidence/parse_details.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/evidence/parse_details.py b/evidence/parse_details.py index 55ecdd7..35adfc5 100644 --- a/evidence/parse_details.py +++ b/evidence/parse_details.py @@ -54,7 +54,7 @@ class ParseSnapshot: } def set_computer(self): - machine = get_inxi_key(self.inxi, 'Machine') + machine = get_inxi_key(self.inxi, 'Machine') or [] for m in machine: system = get_inxi(m, "System") if system: @@ -80,7 +80,7 @@ class ParseSnapshot: self.get_battery() def get_mother_board(self): - machine = get_inxi_key(self.inxi, 'Machine') + machine = get_inxi_key(self.inxi, 'Machine') or [] mb = {"type": "Motherboard",} for m in machine: bios_date = get_inxi(m, "date") @@ -102,7 +102,7 @@ class ParseSnapshot: self.components.append(mb) def get_ram_slots(self, mb): - memory = get_inxi_key(self.inxi, 'Memory') + memory = get_inxi_key(self.inxi, 'Memory') or [] for m in memory: slots = get_inxi(m, "slots") if not slots: @@ -113,7 +113,7 @@ class ParseSnapshot: def get_cpu(self): - cpu = get_inxi_key(self.inxi, 'CPU') + cpu = get_inxi_key(self.inxi, 'CPU') or [] cp = {"type": "Processor"} vulnerabilities = [] for c in cpu: @@ -158,7 +158,7 @@ class ParseSnapshot: def get_ram(self): - memory = get_inxi_key(self.inxi, 'Memory') + memory = get_inxi_key(self.inxi, 'Memory') or [] mem = {"type": "RamModule"} for m in memory: @@ -180,7 +180,7 @@ class ParseSnapshot: self.components.append(mem) def get_graphic(self): - graphics = get_inxi_key(self.inxi, 'Graphics') + graphics = get_inxi_key(self.inxi, 'Graphics') or [] for c in graphics: if not get_inxi(c, "Device") or not get_inxi(c, "vendor"): @@ -199,7 +199,7 @@ class ParseSnapshot: ) def get_battery(self): - bats = get_inxi_key(self.inxi, 'Battery') + bats = get_inxi_key(self.inxi, 'Battery') or [] for b in bats: self.components.append( { @@ -213,7 +213,7 @@ class ParseSnapshot: ) def get_memory_video(self, c): - memory = get_inxi_key(self.inxi, 'Memory') + memory = get_inxi_key(self.inxi, 'Memory') or [] for m in memory: igpu = get_inxi(m, "igpu") @@ -226,7 +226,7 @@ class ParseSnapshot: return self.default def get_data_storage(self): - hdds= get_inxi_key(self.inxi, 'Drives') + hdds= get_inxi_key(self.inxi, 'Drives') or [] for d in hdds: usb = get_inxi(d, "type") if usb == "USB": @@ -277,7 +277,7 @@ class ParseSnapshot: return [] def get_networks(self): - nets = get_inxi_key(self.inxi, "Network") + nets = get_inxi_key(self.inxi, "Network") or [] networks = [(nets[i], nets[i + 1]) for i in range(0, len(nets) - 1, 2)] for n, iface in networks: @@ -306,7 +306,7 @@ class ParseSnapshot: ) def get_sound_card(self): - audio = get_inxi_key(self.inxi, "Audio") + audio = get_inxi_key(self.inxi, "Audio") or [] for c in audio: model = get_inxi(c, "Device") @@ -323,7 +323,7 @@ class ParseSnapshot: ) def get_display(self): - graphics = get_inxi_key(self.inxi, "Graphics") + graphics = get_inxi_key(self.inxi, "Graphics") or [] for c in graphics: if not get_inxi(c, "Monitor"): continue From 19026473379772bab43e2b78efa0c5bb95f0570d Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 20 Nov 2024 18:41:59 +0100 Subject: [PATCH 71/80] fix get_hid --- evidence/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evidence/parse.py b/evidence/parse.py index 9dca596..b65e23c 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -168,7 +168,7 @@ class Build: def get_hid(self, snapshot): try: - self.inxi = json.loads(self.json["inxi"]) + self.inxi = json.loads(self.json["data"]["inxi"]) except Exception: logger.error("No inxi in snapshot %s", self.uuid) return "" From abdefe885e071d5ffffefb1fa7cacc4d5b3ccc27 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 3 Dec 2024 16:37:56 +0100 Subject: [PATCH 72/80] fix parsing --- api/views.py | 16 ++++++++++------ evidence/parse.py | 15 +++++++++++++-- utils/save_snapshots.py | 5 ++++- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/api/views.py b/api/views.py index 0de3e5a..87177fb 100644 --- a/api/views.py +++ b/api/views.py @@ -85,17 +85,21 @@ class NewSnapshotView(ApiMixing): # except Exception: # return JsonResponse({'error': 'Invalid Snapshot'}, status=400) - if not data.get("uuid"): + ev_uuid = data.get("uuid") + if data.get("credentialSubject"): + ev_uuid = data["credentialSubject"].get("uuid") + + if not ev_uuid: txt = "error: the snapshot not have uuid" logger.error("%s", txt) return JsonResponse({'status': txt}, status=500) exist_annotation = Annotation.objects.filter( - uuid=data['uuid'] + uuid=ev_uuid ).first() if exist_annotation: - txt = "error: the snapshot {} exist".format(data['uuid']) + txt = "error: the snapshot {} exist".format(ev_uuid) logger.warning("%s", txt) return JsonResponse({'status': txt}, status=500) @@ -105,14 +109,14 @@ class NewSnapshotView(ApiMixing): except Exception as err: if settings.DEBUG: logger.exception("%s", err) - snapshot_id = data.get("uuid", "") + snapshot_id = ev_uuid txt = "It is not possible to parse snapshot: %s." logger.error(txt, snapshot_id) text = "fail: It is not possible to parse snapshot" return JsonResponse({'status': text}, status=500) annotation = Annotation.objects.filter( - uuid=data['uuid'], + uuid=ev_uuid, type=Annotation.Type.SYSTEM, # TODO this is hardcoded, it should select the user preferred algorithm key="hidalgo1", @@ -121,7 +125,7 @@ class NewSnapshotView(ApiMixing): if not annotation: - logger.error("Error: No annotation for uuid: %s", data["uuid"]) + logger.error("Error: No annotation for uuid: %s", ev_uuid) return JsonResponse({'status': 'fail'}, status=500) url_args = reverse_lazy("device:details", args=(annotation.value,)) diff --git a/evidence/parse.py b/evidence/parse.py index b65e23c..d30e88e 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -32,7 +32,18 @@ def get_mac(inxi): class Build: def __init__(self, evidence_json, user, check=False): - self.json = evidence_json + self.evidence = evidence_json.copy() + self.json = evidence_json.copy() + if evidence_json.get("credentialSubject"): + self.json.update(evidence_json["credentialSubject"]) + if evidence_json.get("evidence"): + self.json["data"] = {} + for ev in evidence_json["evidence"]: + k = ev.get("operation") + if not k: + continue + self.json["data"][k] = ev.get("output") + self.uuid = self.json['uuid'] self.user = user self.hid = None @@ -49,7 +60,7 @@ class Build: self.register_device_dlt() def index(self): - snap = json.dumps(self.json) + snap = json.dumps(self.evidence) index(self.user.institution, self.uuid, snap) def generate_chids(self): diff --git a/utils/save_snapshots.py b/utils/save_snapshots.py index 8c02f06..efd1fe9 100644 --- a/utils/save_snapshots.py +++ b/utils/save_snapshots.py @@ -19,7 +19,10 @@ def move_json(path_name, user, place="snapshots"): def save_in_disk(data, user, place="snapshots"): - uuid = data.get('uuid', '') + uuid = data.get("uuid") + if data.get("credentialSubject"): + uuid = data["credentialSubject"].get("uuid") + now = datetime.now() year = now.year month = now.month From 46bbb940d7af2ece7659773bdff9f065a2f91f9d Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 5 Dec 2024 19:23:53 +0100 Subject: [PATCH 73/80] fix parsing with credentials --- dashboard/views.py | 7 ++++-- evidence/models.py | 53 +++++++++++++++++++++++++-------------- evidence/parse.py | 5 +++- evidence/parse_details.py | 25 ++++++++++++------ 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/dashboard/views.py b/dashboard/views.py index 8d91732..2ec12a0 100644 --- a/dashboard/views.py +++ b/dashboard/views.py @@ -84,8 +84,11 @@ class SearchView(InventaryMixin): return devices, count def get_annotations(self, xp): - snap = xp.document.get_data() - uuid = json.loads(snap).get('uuid') + snap = json.loads(xp.document.get_data()) + if snap.get("credentialSubject"): + uuid = snap["credentialSubject"]["uuid"] + else: + uuid = snap["uuid"] return Device.get_annotation_from_uuid(uuid, self.request.user.institution) def search_hids(self, query, offset, limit): diff --git a/evidence/models.py b/evidence/models.py index 99e6a96..b410795 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -78,25 +78,37 @@ class Evidence: for xa in matches: self.doc = json.loads(xa.document.get_data()) - if not self.is_legacy(): - dmidecode_raw = self.doc["data"]["dmidecode"] - inxi_raw = self.doc["data"]["inxi"] - self.dmi = DMIParse(dmidecode_raw) - try: - self.inxi = json.loads(inxi_raw) - machine = get_inxi_key(self.inxi, 'Machine') - for m in machine: - system = get_inxi(m, "System") - if system: - self.device_manufacturer = system - self.device_model = get_inxi(m, "product") - self.device_serial_number = get_inxi(m, "serial") - self.device_chassis = get_inxi(m, "Type") - self.device_version = get_inxi(m, "v") + if self.is_legacy(): + return + if self.doc.get("credentialSubject"): + for ev in self.doc["evidence"]: + if "dmidecode" == ev.get("operation"): + dmidecode_raw = ev["output"] + if "inxi" == ev.get("operation"): + self.inxi = ev["output"] + else: + dmidecode_raw = self.doc["data"]["dmidecode"] + try: + self.inxi = json.loads(self.doc["data"]["inxi"]) except Exception: return + self.dmi = DMIParse(dmidecode_raw) + try: + machine = get_inxi_key(self.inxi, 'Machine') + for m in machine: + system = get_inxi(m, "System") + if system: + self.device_manufacturer = system + self.device_model = get_inxi(m, "product") + self.device_serial_number = get_inxi(m, "serial") + self.device_chassis = get_inxi(m, "Type") + self.device_version = get_inxi(m, "v") + + except Exception: + return + def get_time(self): if not self.doc: self.get_doc() @@ -123,7 +135,7 @@ class Evidence: if self.inxi: return self.device_manufacturer - + return self.dmi.manufacturer().strip() def get_model(self): @@ -138,13 +150,13 @@ class Evidence: if self.inxi: return self.device_model - + return self.dmi.model().strip() def get_chassis(self): if self.is_legacy(): return self.doc['device']['model'] - + if self.inxi: return self.device_chassis @@ -159,7 +171,7 @@ class Evidence: def get_serial_number(self): if self.is_legacy(): return self.doc['device']['serialNumber'] - + if self.inxi: return self.device_serial_number @@ -185,6 +197,9 @@ class Evidence: self.components = snapshot['components'] def is_legacy(self): + if self.doc.get("credentialSubject"): + return False + return self.doc.get("software") != "workbench-script" def is_web_snapshot(self): diff --git a/evidence/parse.py b/evidence/parse.py index d30e88e..790f809 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -34,6 +34,7 @@ class Build: def __init__(self, evidence_json, user, check=False): self.evidence = evidence_json.copy() self.json = evidence_json.copy() + if evidence_json.get("credentialSubject"): self.json.update(evidence_json["credentialSubject"]) if evidence_json.get("evidence"): @@ -179,7 +180,9 @@ class Build: def get_hid(self, snapshot): try: - self.inxi = json.loads(self.json["data"]["inxi"]) + self.inxi = self.json["data"]["inxi"] + if isinstance(self.inxi, str): + self.inxi = json.loads(self.inxi) except Exception: logger.error("No inxi in snapshot %s", self.uuid) return "" diff --git a/evidence/parse_details.py b/evidence/parse_details.py index 35adfc5..7ce0a5b 100644 --- a/evidence/parse_details.py +++ b/evidence/parse_details.py @@ -30,9 +30,20 @@ def get_inxi(n, name): class ParseSnapshot: def __init__(self, snapshot, default="n/a"): self.default = default - self.dmidecode_raw = snapshot["data"].get("dmidecode", "{}") - self.smart_raw = snapshot["data"].get("disks", []) - self.inxi_raw = snapshot["data"].get("inxi", "") or "" + self.dmidecode_raw = snapshot.get("data", {}).get("dmidecode", "{}") + self.smart_raw = snapshot.get("data", {}).get("smartctl", []) + self.inxi_raw = snapshot.get("data", {}).get("inxi", "") or "" + for ev in snapshot.get("evidence", []): + if "dmidecode" == ev.get("operation"): + self.dmidecode_raw = ev["output"] + if "inxi" == ev.get("operation"): + self.inxi_raw = ev["output"] + if "smartctl" == ev.get("operation"): + self.smart_raw = ev["output"] + data = snapshot + if snapshot.get("credentialSubject"): + data = snapshot["credentialSubject"] + self.device = {"actions": []} self.components = [] @@ -45,11 +56,10 @@ class ParseSnapshot: self.snapshot_json = { "type": "Snapshot", "device": self.device, - "software": snapshot["software"], + "software": data["software"], "components": self.components, - "uuid": snapshot['uuid'], - "version": snapshot['version'], - "endTime": snapshot["timestamp"], + "uuid": data['uuid'], + "endTime": data["timestamp"], "elapsed": 1, } @@ -267,7 +277,6 @@ class ParseSnapshot: hd["read used"] = get_inxi(d, "read-units") hd["written used"] = get_inxi(d, "written-units") - # import pdb; pdb.set_trace() self.components.append(hd) continue From 782f6dac51c3f916862128fa17a26743a56677ed Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 16 Dec 2024 18:12:02 +0100 Subject: [PATCH 74/80] refactor inxi --- evidence/forms.py | 12 ++++++------ evidence/models.py | 38 ++++++++++++++++++++------------------ evidence/parse.py | 43 +++++-------------------------------------- 3 files changed, 31 insertions(+), 62 deletions(-) diff --git a/evidence/forms.py b/evidence/forms.py index 83d9e6b..8be2887 100644 --- a/evidence/forms.py +++ b/evidence/forms.py @@ -29,17 +29,17 @@ class UploadForm(forms.Form): try: file_json = json.loads(file_data) - Build(file_json, None, check=True) + snap = Build(file_json, None, check=True) exist_annotation = Annotation.objects.filter( - uuid=file_json['uuid'] + uuid=snap.uuid ).first() - + if exist_annotation: - raise ValidationError( + raise ValidationError( _("The snapshot already exists"), code="duplicate_snapshot", ) - + #Catch any error and display it as Validation Error so the Form handles it except Exception as e: raise ValidationError( @@ -221,7 +221,7 @@ class EraseServerForm(forms.Form): if self.instance: return - + Annotation.objects.create( uuid=self.uuid, type=Annotation.Type.ERASE_SERVER, diff --git a/evidence/models.py b/evidence/models.py index b410795..a10de01 100644 --- a/evidence/models.py +++ b/evidence/models.py @@ -63,13 +63,16 @@ class Evidence: def get_phid(self): if not self.doc: self.get_doc() - + return hashlib.sha3_256(json.dumps(self.doc)).hexdigest() def get_doc(self): self.doc = {} + self.inxi = None + if not self.owner: self.get_owner() + qry = 'uuid:"{}"'.format(self.uuid) matches = search(self.owner, qry, limit=1) if matches and matches.size() < 0: @@ -89,26 +92,26 @@ class Evidence: self.inxi = ev["output"] else: dmidecode_raw = self.doc["data"]["dmidecode"] + inxi_raw = self.doc["data"]["inxi"] + self.dmi = DMIParse(dmidecode_raw) try: - self.inxi = json.loads(self.doc["data"]["inxi"]) + self.inxi = json.loads(inxi_raw) + except Exception: + pass + if self.inxi: + try: + machine = get_inxi_key(self.inxi, 'Machine') + for m in machine: + system = get_inxi(m, "System") + if system: + self.device_manufacturer = system + self.device_model = get_inxi(m, "product") + self.device_serial_number = get_inxi(m, "serial") + self.device_chassis = get_inxi(m, "Type") + self.device_version = get_inxi(m, "v") except Exception: return - self.dmi = DMIParse(dmidecode_raw) - try: - machine = get_inxi_key(self.inxi, 'Machine') - for m in machine: - system = get_inxi(m, "System") - if system: - self.device_manufacturer = system - self.device_model = get_inxi(m, "product") - self.device_serial_number = get_inxi(m, "serial") - self.device_chassis = get_inxi(m, "Type") - self.device_version = get_inxi(m, "v") - - except Exception: - return - def get_time(self): if not self.doc: self.get_doc() @@ -177,7 +180,6 @@ class Evidence: return self.dmi.serial_number().strip() - def get_version(self): if self.inxi: return self.device_version diff --git a/evidence/parse.py b/evidence/parse.py index 790f809..db2d32d 100644 --- a/evidence/parse.py +++ b/evidence/parse.py @@ -3,21 +3,16 @@ import hashlib import logging from dmidecode import DMIParse -from json_repair import repair_json -from django.conf import settings -from evidence.parse_details import get_lshw_child, ParseSnapshot +from evidence.parse_details import ParseSnapshot from evidence.models import Annotation from evidence.xapian import index -from utils.constants import CHASSIS_DH +from evidence.parse_details import get_inxi_key, get_inxi +from django.conf import settings if settings.DPP: from dpp.api_dlt import register_device_dlt, register_passport_dlt - - -from evidence.parse_details import get_inxi_key, get_inxi - logger = logging.getLogger('django') @@ -82,27 +77,16 @@ class Build: sku = device.get("sku", '') hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}" - self.chid = hashlib.sha3_256(hid.encode()).hexdigest() return self.chid def get_chid_dpp(self): if self.json.get("software") == "workbench-script": - dmidecode_raw = self.json["data"]["dmidecode"] - dmi = DMIParse(dmidecode_raw) - - manufacturer = dmi.manufacturer().strip() - model = dmi.model().strip() - chassis = self.get_chassis_dh() - serial_number = dmi.serial_number() - sku = self.get_sku() - typ = chassis - version = self.get_version() - hid = f"{manufacturer}{model}{chassis}{serial_number}{sku}{typ}{version}" + device = ParseSnapshot(self.json).device else: device = self.json['device'] - hid = self.get_id_hw_dpp(device) + hid = self.get_id_hw_dpp(device) self.chid = hashlib.sha3_256(hid.encode("utf-8")).hexdigest() return self.chid @@ -161,23 +145,6 @@ class Build: value=v ) - def get_chassis_dh(self): - chassis = self.get_chassis() - lower_type = chassis.lower() - for k, v in CHASSIS_DH.items(): - if lower_type in v: - return k - return self.default - - def get_sku(self): - return self.dmi.get("System")[0].get("SKU Number", "n/a").strip() - - def get_chassis(self): - return self.dmi.get("Chassis")[0].get("Type", '_virtual') # - - def get_version(self): - return self.dmi.get("System")[0].get("Verson", '_virtual') - def get_hid(self, snapshot): try: self.inxi = self.json["data"]["inxi"] From e6fea8c1c33f17dedbd36f8c6a0a37bce1520aa7 Mon Sep 17 00:00:00 2001 From: pedro Date: Mon, 16 Dec 2024 19:05:08 +0100 Subject: [PATCH 75/80] update requirements so works on lxc-docker --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 52409e6..f12a20c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ xlrd==2.0.1 odfpy==1.4.1 pytz==2024.2 json-repair==0.30.0 -setuptools==75.5.0 +setuptools==65.5.1 requests==2.32.3 -wheel==0.45.0 +wheel==0.45.1 + From d8202fea0faf316feed7b0ec4cd5af2ec8c84988 Mon Sep 17 00:00:00 2001 From: pedro Date: Mon, 16 Dec 2024 19:06:11 +0100 Subject: [PATCH 76/80] dockerfile: remove unnecessary pip upgrade --- docker/devicehub-django.Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/devicehub-django.Dockerfile b/docker/devicehub-django.Dockerfile index 6f5ec95..0f0d408 100644 --- a/docker/devicehub-django.Dockerfile +++ b/docker/devicehub-django.Dockerfile @@ -28,9 +28,6 @@ compile = no no-cache-dir = True END -# upgrade pip, which might fail on lxc, then remove the "corrupted file" -RUN python -m pip install --upgrade pip || (rm -rf /usr/local/lib/python3.11/site-packages/pip-*.dist-info && python -m pip install --upgrade pip) - COPY ./requirements.txt /opt/devicehub-django RUN pip install -r requirements.txt # TODO hardcoded, is ignored in requirements.txt From 1d195806aaf42bda0ead4de5212fd441a5962654 Mon Sep 17 00:00:00 2001 From: pedro Date: Mon, 16 Dec 2024 19:25:32 +0100 Subject: [PATCH 77/80] bad bugfix: docker entrypoint overrides .env because of dpp --- docker/devicehub-django.entrypoint.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index d6d7bb2..a6027db 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -50,14 +50,13 @@ API_RESOLVER=${API_RESOLVER} ID_FEDERATED=${ID_FEDERATED} END )" - fi - # generate config using env vars from docker # TODO rethink if this is needed because now this is django, not flask cat > .env < Date: Mon, 16 Dec 2024 19:28:08 +0100 Subject: [PATCH 78/80] better .env.example (just edit DOMAIN) --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index d8fed15..6fb20c9 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ DOMAIN=localhost DEMO=true # note that with DEBUG=true, logs are more verbose (include tracebacks) DEBUG=true -ALLOWED_HOSTS=localhost,localhost:8000,127.0.0.1, +ALLOWED_HOSTS=${DOMAIN},${DOMAIN}:8000,127.0.0.1,127.0.0.1:8000 DPP=false STATIC_ROOT=/tmp/static/ From 67213e46d269c93dfb3a8c8ea20f4f6b0aad4594 Mon Sep 17 00:00:00 2001 From: pedro Date: Mon, 16 Dec 2024 19:29:48 +0100 Subject: [PATCH 79/80] bugfix typo on entrypoint --- docker/devicehub-django.entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/devicehub-django.entrypoint.sh b/docker/devicehub-django.entrypoint.sh index a6027db..1b9f548 100644 --- a/docker/devicehub-django.entrypoint.sh +++ b/docker/devicehub-django.entrypoint.sh @@ -55,8 +55,8 @@ END cat > .env < Date: Mon, 16 Dec 2024 19:37:59 +0100 Subject: [PATCH 80/80] docker-reset: bugfix to reset, add flag file --- docker-reset.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-reset.sh b/docker-reset.sh index 825e91c..8bbc229 100755 --- a/docker-reset.sh +++ b/docker-reset.sh @@ -21,6 +21,8 @@ main() { fi # remove old database rm -vfr ./db/* + # deactivate configured flag + rm -vfr ./already_configured docker compose down -v docker compose build docker compose up ${detach_arg:-}