From 95beec7d930830679a81df7cd8536f08357db47b Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 14 Feb 2024 10:46:31 +0100 Subject: [PATCH 1/7] return old fix --- idhub/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/idhub/models.py b/idhub/models.py index 8f074bb..aa2ac7c 100644 --- a/idhub/models.py +++ b/idhub/models.py @@ -600,7 +600,7 @@ class VerificableCredential(models.Model): if not self.data: return "" - if self.eidas1_did or self.is_didweb: + if self.eidas1_did: return self.data return self.user.decrypt_data(self.data, password) @@ -646,7 +646,7 @@ class VerificableCredential(models.Model): self.render(domain), self.issuer_did.get_key_material(issuer_pass) ) - if self.eidas1_did or self.is_didweb: + if self.eidas1_did: self.data = data else: self.data = self.user.encrypt_data(data, password) @@ -660,7 +660,7 @@ class VerificableCredential(models.Model): cred_path = 'credentials' sid = self.id - if self.eidas1_did or self.is_didweb: + if self.eidas1_did: cred_path = 'public/credentials' sid = self.hash From eb1abc89b46636d0549f42f078db3bd60ff6df0b Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 14 Feb 2024 10:59:41 +0100 Subject: [PATCH 2/7] remove restrictions of unique File_datas --- idhub/admin/forms.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/idhub/admin/forms.py b/idhub/admin/forms.py index 59d0eb3..581c2b0 100644 --- a/idhub/admin/forms.py +++ b/idhub/admin/forms.py @@ -195,8 +195,6 @@ class ImportForm(forms.Form): def clean_file_import(self): data = self.cleaned_data["file_import"] self.file_name = data.name - if File_datas.objects.filter(file_name=self.file_name, success=True).exists(): - raise ValidationError("This file already exists!") # df = pd.read_csv (data, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL) df = pd.read_excel(data) From 14167090b3608a669e0333ae35dff99129b34900 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 14 Feb 2024 12:59:08 +0100 Subject: [PATCH 3/7] update eOperatorClaim schema --- schemas/e-operator-claim.json | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/schemas/e-operator-claim.json b/schemas/e-operator-claim.json index 59b2434..cc6e0c9 100644 --- a/schemas/e-operator-claim.json +++ b/schemas/e-operator-claim.json @@ -36,6 +36,14 @@ "description": "Legal name of the operator", "type": "string" }, + "role": { + "description": "Role, either operator, witness, auditor", + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, "accreditedBy": { "description": "Legal name of the accrediting entity", "type": "string" @@ -51,16 +59,13 @@ "accreditedFor": { "description": "Operation type: e.g. manufacture, repair, refurbishment, remanufacture, transport, dismantle, recycle, verification, audit", "type": "string" - }, - "role": { - "description": "Role, either operator, witness, auditor", - "type": "string" } }, "required": [ "id", "legalName", - "role" + "role", + "email" ] } } From 3289b99b6e7eb5f8d24e847077c84636904e82fd Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 14 Feb 2024 12:59:43 +0100 Subject: [PATCH 4/7] fix issue credential next to pass didkit --- idhub/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/idhub/models.py b/idhub/models.py index aa2ac7c..1412f00 100644 --- a/idhub/models.py +++ b/idhub/models.py @@ -631,7 +631,6 @@ class VerificableCredential(models.Model): if self.status == self.Status.ISSUED: return - self.status = self.Status.ISSUED self.subject_did = did self.issued_on = datetime.datetime.now().astimezone(pytz.utc) issuer_pass = cache.get("KEY_DIDS") @@ -651,6 +650,8 @@ class VerificableCredential(models.Model): else: self.data = self.user.encrypt_data(data, password) + self.status = self.Status.ISSUED + def get_context(self, domain): d = json.loads(self.csv_data) issuance_date = '' From c218245707dbf31a431f4a08141be08a1817daa0 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 14 Feb 2024 13:01:43 +0100 Subject: [PATCH 5/7] filter fields of csv and no repeat credential --- idhub/admin/forms.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/idhub/admin/forms.py b/idhub/admin/forms.py index 581c2b0..422d309 100644 --- a/idhub/admin/forms.py +++ b/idhub/admin/forms.py @@ -196,7 +196,6 @@ class ImportForm(forms.Form): data = self.cleaned_data["file_import"] self.file_name = data.name - # df = pd.read_csv (data, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL) df = pd.read_excel(data) data_pd = df.fillna('').to_dict() @@ -205,9 +204,10 @@ class ImportForm(forms.Form): head_row = {x: '' for x in self.properties.keys()} for n in range(df.last_valid_index()+1): - row = head_row.copy() + row = {} for k in data_pd.keys(): - row[k] = data_pd[k][n] or '' + if k in self.properties.keys(): + row[k] = data_pd[k][n] or '' user = self.validate_jsonld(n, row) self.rows[user] = row @@ -229,12 +229,13 @@ class ImportForm(forms.Form): def validate_jsonld(self, line, row): try: - check = credtools.validate_json(row, self.json_schema_filtered) + # check = credtools.validate_json(row, self.json_schema_filtered) + check = credtools.validate_json(row, self.json_schema) if check is not True: raise ValidationError("Not valid row") except Exception as e: msg = "line {}: {}".format(line+1, e) - self.exception(msg) + return self.exception(msg) user, new = User.objects.get_or_create(email=row.get('email')) if new: @@ -243,6 +244,18 @@ class ImportForm(forms.Form): return user def create_credential(self, user, row): + bcred = VerificableCredential.objects.filter( + user=user, + schema=self._schema, + issuer_did=self._did, + status=VerificableCredential.Status.ENABLED + ) + if bcred.exists(): + cred = bcred.first() + cred.csv_data = json.dumps(row, default=str) + cred.eidas1_did = self._eidas1 + return cred + cred = VerificableCredential( verified=False, user=user, From 5aa22419ac4b86e006c6d87b16032e162825d453 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 15 Feb 2024 18:59:57 +0100 Subject: [PATCH 6/7] fix validate credentials --- idhub/admin/forms.py | 37 ++++++++++++-------------- idhub/models.py | 22 ++++++++++++++++ utils/credtools.py | 62 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 96 insertions(+), 25 deletions(-) diff --git a/idhub/admin/forms.py b/idhub/admin/forms.py index 422d309..aa453ce 100644 --- a/idhub/admin/forms.py +++ b/idhub/admin/forms.py @@ -1,7 +1,8 @@ import csv import json -import base64 import copy +import base64 +import jsonschema import pandas as pd from pyhanko.sign import signers @@ -176,20 +177,10 @@ class ImportForm(forms.Form): self._schema = schema.first() try: - self.json_schema = json.loads(self._schema.data) - props = [x for x in self.json_schema["allOf"] if 'properties' in x.keys()] - prop = props[0]['properties'] - self.properties = prop['credentialSubject']['properties'] + self.json_schema = self._schema.get_credential_subject_schema() except Exception: raise ValidationError("Schema is not valid!") - if not self.properties: - raise ValidationError("Schema is not valid!") - - # TODO we need filter "$ref" of schema for can validate a csv - self.json_schema_filtered = copy.copy(self.json_schema) - allOf = [x for x in self.json_schema["allOf"] if '$ref' not in x.keys()] - self.json_schema_filtered["allOf"] = allOf return data def clean_file_import(self): @@ -197,17 +188,19 @@ class ImportForm(forms.Form): self.file_name = data.name df = pd.read_excel(data) + # convert dates to iso 8601 + for col in df.select_dtypes(include='datetime').columns: + df[col] = df[col].dt.strftime("%Y-%m-%d") + data_pd = df.fillna('').to_dict() if not data_pd: self.exception("This file is empty!") - head_row = {x: '' for x in self.properties.keys()} for n in range(df.last_valid_index()+1): row = {} for k in data_pd.keys(): - if k in self.properties.keys(): - row[k] = data_pd[k][n] or '' + row[k] = data_pd[k][n] or '' user = self.validate_jsonld(n, row) self.rows[user] = row @@ -229,13 +222,15 @@ class ImportForm(forms.Form): def validate_jsonld(self, line, row): try: - # check = credtools.validate_json(row, self.json_schema_filtered) - check = credtools.validate_json(row, self.json_schema) - if check is not True: - raise ValidationError("Not valid row") - except Exception as e: - msg = "line {}: {}".format(line+1, e) + jsonschema.validate(instance=row, schema=self.json_schema) + except jsonschema.exceptions.ValidationError as err: + msg = "line {}: {}".format(line+1, err) return self.exception(msg) + # try: + # check = credtools.validate_json(row, self.json_schema) + # if check is not True: + # raise ValidationError("Not valid row") + # except Exception as e: user, new = User.objects.get_or_create(email=row.get('email')) if new: diff --git a/idhub/models.py b/idhub/models.py index 1412f00..a9bf727 100644 --- a/idhub/models.py +++ b/idhub/models.py @@ -541,6 +541,28 @@ class Schemas(models.Model): def description(self, value): self._description = value + def get_credential_subject_schema(self): + sc = self.get_data() + properties = sc["allOf"][1]["properties"]["credentialSubject"]["properties"] + required = sc["allOf"][1]["properties"]["credentialSubject"]["required"] + + if "id" in required: + required.remove("id") + + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": properties, + "required": required, + "additionalProperties": False + } + + return schema + + def get_data(self): + return json.loads(self.data) + + class VerificableCredential(models.Model): """ Definition of Verificable Credentials diff --git a/utils/credtools.py b/utils/credtools.py index c077758..642b96f 100644 --- a/utils/credtools.py +++ b/utils/credtools.py @@ -9,6 +9,7 @@ import requests from pyld import jsonld import jsonref from jsonpath_ng import jsonpath, parse +from datetime import datetime # def remove_null_values(dictionary): @@ -205,22 +206,75 @@ def schema_to_xls_comment(schema, xls_file_path): # Get the xlsxwriter workbook and worksheet objects workbook = writer.book + + matches_title = parse('$.title').find(schema) + title = matches_title[0].value if matches_title else 'no title' + + matches_desc = parse('$.description').find(schema) + desc = matches_desc[0].value if matches_desc else 'no description' + + matches_id = parse("$['$id']").find(schema) + idschema = matches_id[0].value if matches_id else 'no schema' + + matches_subject_desc = parse('$..credentialSubject.description').find(schema) + subject_desc = matches_subject_desc[0].value if matches_subject_desc else 'no subject description' + + workbook.set_properties({ + 'title': title, + 'subject': desc, + 'author': 'IdHub Orchestral', + 'category': subject_desc, + 'keywords': 'schema, template, plantilla', + 'created': datetime.now().date(), #datetime.date(2018, 1, 1), + 'comments': 'Created with Python for IdHub'}) + + workbook.set_custom_property('Schema', idschema) + worksheet = writer.sheets['Full1'] # Define a format for the required header cells - req_format = workbook.add_format({'border': 1}) - # cell_format = workbook.add_format({'bold': True, 'font_color': 'red'}) + req_f = workbook.add_format({'border': 1}) + req_da = workbook.add_format({'border': 1, 'num_format': 'yyyy-mm-dd'}) + req_in = workbook.add_format({'border': 1, 'num_format': '0'}) + req_st = workbook.add_format({'border': 1, 'num_format': '@'}) + opt_da = workbook.add_format({'num_format': 'yyyy-mm-dd'}) + opt_in = workbook.add_format({'num_format': '0'}) + opt_st = workbook.add_format({'num_format': '@'}) + fmts = { + 'string' : {True: req_st, False: opt_st}, + 'date' : {True: req_da, False: opt_da}, + 'integer' : {True: req_in, False: opt_in} + } # Write comments to the cells for i, header in enumerate(headers): - if header in req: - worksheet.set_column(i,i, None, req_format) + fmt = {} + #if header in req: + # fmt = req_format + # worksheet.set_column(i,i, None, req_format) + # Get the description for the current field if 'description' in matches[0][header]: description = matches[0][header]['description'] if description is not None: # Write the description as a comment to the corresponding cell worksheet.write_comment(0, i, description) + + # Get the type for the current field + if 'type' in matches[0][header]: + type_field = matches[0][header]['type'] + + format_field = None + if 'format' in matches[0][header]: + format_field = matches[0][header]['format'] + + if type_field is not None: + if format_field is not None and format_field == 'date': + type_field = 'date' + fmt = fmts[type_field][header in req] # Add type format + + print(f'header {header} with fmt {fmt}\n') + worksheet.set_column(i,i, None, fmt) # Close the Pandas Excel writer and output the Excel file worksheet.autofit() From bb54eb039229caa3ac12618b1cb22851ef737c62 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 16 Feb 2024 00:25:04 +0100 Subject: [PATCH 7/7] verification after signed credential --- idhub/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/idhub/models.py b/idhub/models.py index 7e6f847..ae61c56 100644 --- a/idhub/models.py +++ b/idhub/models.py @@ -16,6 +16,7 @@ from utils.idhub_ssikit import ( keydid_from_controller_key, sign_credential, webdid_from_controller_key, + verify_credential, ) from idhub_auth.models import User @@ -669,6 +670,10 @@ class VerificableCredential(models.Model): self.render(domain), self.issuer_did.get_key_material(issuer_pass) ) + valid, reason = verify_credential(data) + if not valid: + return + if self.eidas1_did: self.data = data else: