From 091e88fb6abe2bf5351e7b679a6400f58022684a Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Dec 2020 16:24:11 +0100 Subject: [PATCH 1/8] Modify Changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82e952b5..97e2c647 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ml). ## master - [1.0.1-beta] + [1.0.2-beta] ## testing [1.0.3-beta] ## [1.0.3-beta] - [addend] #85 add mac of network adapter to device hid +- [addend] #95 adding endpoint for check the hash of one report - [changed] #94 change form of snapshot manual ## [1.0.2-beta] From d8c7d553668ac319def1b2b00e4bb9af16d08130 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Dec 2020 16:45:08 +0100 Subject: [PATCH 2/8] alembic for define the hash table --- .../versions/3eb50297c365_add_hash.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py diff --git a/ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py b/ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py new file mode 100644 index 00000000..32be4e8d --- /dev/null +++ b/ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py @@ -0,0 +1,42 @@ +"""empty message + +Revision ID: 3eb50297c365 +Revises: 378b6b147b46 +Create Date: 2020-12-18 16:26:15.453694 + +""" +from alembic import context +from alembic import op +import sqlalchemy as sa +import sqlalchemy_utils +import citext +import teal + + +# revision identifiers, used by Alembic. +revision = '3eb50297c365' +down_revision = '378b6b147b46' +branch_labels = None +depends_on = None + + +def get_inv(): + INV = context.get_x_argument(as_dictionary=True).get('inventory') + if not INV: + raise ValueError("Inventory value is not specified") + return INV + +def upgrade(): + # Report Hash table + op.create_table('report_hash', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('created', sa.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), + nullable=False, comment='When Devicehub created this.'), + sa.Column('hash', citext.CIText(), nullable=False), + sa.PrimaryKeyConstraint('id'), + schema=f'{get_inv()}' + ) + + +def downgrade(): + op.drop_table('report_hash', schema=f'{get_inv()}') From db92c390d9155f2c4623f288275c58be3ad69980 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Dec 2020 20:10:58 +0100 Subject: [PATCH 3/8] fixed migration and insert_hash function --- .../versions/3eb50297c365_add_hash.py | 15 +++++----- ereuse_devicehub/resources/hash_reports.py | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 ereuse_devicehub/resources/hash_reports.py diff --git a/ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py b/ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py index 32be4e8d..e6eff98d 100644 --- a/ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py +++ b/ereuse_devicehub/migrations/versions/3eb50297c365_add_hash.py @@ -5,12 +5,13 @@ Revises: 378b6b147b46 Create Date: 2020-12-18 16:26:15.453694 """ -from alembic import context -from alembic import op -import sqlalchemy as sa -import sqlalchemy_utils + import citext -import teal +import sqlalchemy as sa + +from alembic import op +from alembic import context +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. @@ -29,10 +30,10 @@ def get_inv(): def upgrade(): # Report Hash table op.create_table('report_hash', - sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), sa.Column('created', sa.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False, comment='When Devicehub created this.'), - sa.Column('hash', citext.CIText(), nullable=False), + sa.Column('hash3', citext.CIText(), nullable=False), sa.PrimaryKeyConstraint('id'), schema=f'{get_inv()}' ) diff --git a/ereuse_devicehub/resources/hash_reports.py b/ereuse_devicehub/resources/hash_reports.py new file mode 100644 index 00000000..26bf5bd2 --- /dev/null +++ b/ereuse_devicehub/resources/hash_reports.py @@ -0,0 +1,29 @@ +"""Hash implementation and save in database +""" +import hashlib + +from citext import CIText +from sqlalchemy import Column +from sqlalchemy.dialects.postgresql import UUID +from uuid import uuid4 + +from ereuse_devicehub.db import db + + +class ReportHash(db.Model): + """Save the hash than is create when one report is download. + """ + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid4) + id.comment = """The identifier of the device for this database. Used only + internally for software; users should not use this. + """ + hash3 = db.Column(CIText(), nullable=False) + hash3.comment = """The normalized name of the hash.""" + + +def insert_hash(bfile=b'hello'): + hash3 = hashlib.sha3_256(bfile).hexdigest() + db_hash = ReportHash(hash3=hash3) + db.session.add(db_hash) + db.session.commit() + db.session.flush() From db19f90009e585fc95a2d71ed20601914f51b6e0 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 18 Dec 2020 20:11:50 +0100 Subject: [PATCH 4/8] clean --- ereuse_devicehub/resources/hash_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ereuse_devicehub/resources/hash_reports.py b/ereuse_devicehub/resources/hash_reports.py index 26bf5bd2..8822afca 100644 --- a/ereuse_devicehub/resources/hash_reports.py +++ b/ereuse_devicehub/resources/hash_reports.py @@ -21,7 +21,7 @@ class ReportHash(db.Model): hash3.comment = """The normalized name of the hash.""" -def insert_hash(bfile=b'hello'): +def insert_hash(bfile): hash3 = hashlib.sha3_256(bfile).hexdigest() db_hash = ReportHash(hash3=hash3) db.session.add(db_hash) From db610d28534ff729034172a1fe5ad07c9c249deb Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Dec 2020 11:34:03 +0100 Subject: [PATCH 5/8] insert hash3 when document is download --- .../resources/documents/documents.py | 5 ++++- tests/test_documents.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index e65bee81..822adbf6 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -23,6 +23,7 @@ from ereuse_devicehub.resources.documents.device_row import DeviceRow, StockRow from ereuse_devicehub.resources.documents.device_row import DeviceRow from ereuse_devicehub.resources.lot import LotView from ereuse_devicehub.resources.lot.models import Lot +from ereuse_devicehub.resources.hash_reports import insert_hash @@ -125,7 +126,9 @@ class DevicesDocumentView(DeviceView): cw.writerow(d.keys()) first = False cw.writerow(d.values()) - output = make_response(data.getvalue().encode('utf-8')) + bfile = data.getvalue().encode('utf-8') + insert_hash(bfile) + output = make_response(bfile) output.headers['Content-Disposition'] = 'attachment; filename=export.csv' output.headers['Content-type'] = 'text/csv' return output diff --git a/tests/test_documents.py b/tests/test_documents.py index 281ca65a..162ed5a4 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -1,4 +1,5 @@ import csv +import hashlib from datetime import datetime from io import StringIO from pathlib import Path @@ -15,7 +16,9 @@ from ereuse_devicehub.resources.documents import documents from ereuse_devicehub.resources.device import models as d from ereuse_devicehub.resources.lot.models import Lot from ereuse_devicehub.resources.tag.model import Tag +from ereuse_devicehub.resources.hash_reports import ReportHash from ereuse_devicehub.db import db +from tests import conftest from tests.conftest import file @@ -103,6 +106,7 @@ def test_export_csv_permitions(user: UserClient, user2: UserClient, client: Clie assert len(csv_user) > 0 assert len(csv_user2) == 0 + @pytest.mark.mvp def test_export_basic_snapshot(user: UserClient): """Test export device information in a csv file.""" @@ -128,6 +132,20 @@ def test_export_basic_snapshot(user: UserClient): assert fixture_csv[1][18] == export_csv[1][18], 'Computer information are not equal' assert fixture_csv[1][20:] == export_csv[1][20:], 'Computer information are not equal' + +@pytest.mark.mvp +@pytest.mark.usefixtures(conftest.app_context.__name__) +def test_check_insert_hash(app: Devicehub, user: UserClient): + """Test export device information in a csv file.""" + snapshot, _ = user.post(file('basic.snapshot'), res=Snapshot) + csv_str, _ = user.get(res=documents.DocumentDef.t, + item='devices/', + accept='text/csv', + query=[('filter', {'type': ['Computer']})]) + hash3 = hashlib.sha3_256(csv_str.encode('utf-8')).hexdigest() + assert ReportHash.query.filter_by(hash3=hash3).count() == 1 + + @pytest.mark.mvp def test_export_extended(app: Devicehub, user: UserClient): """Test a export device with all information and a lot of components.""" From 3c12d1f75cf0bda954cf687da5abe5893df6f78d Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Dec 2020 13:40:07 +0100 Subject: [PATCH 6/8] add endpoint --- .../resources/documents/documents.py | 25 +++++++++++++++---- tests/test_documents.py | 3 +++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index 822adbf6..fb5572bd 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -11,20 +11,19 @@ import flask import flask_weasyprint import teal.marshmallow from boltons import urlutils -from flask import make_response, g +from flask import make_response, g, request +from flask.json import jsonify from teal.cache import cache -from teal.resource import Resource +from teal.resource import Resource, View from ereuse_devicehub.db import db from ereuse_devicehub.resources.action import models as evs from ereuse_devicehub.resources.device import models as devs from ereuse_devicehub.resources.device.views import DeviceView from ereuse_devicehub.resources.documents.device_row import DeviceRow, StockRow -from ereuse_devicehub.resources.documents.device_row import DeviceRow from ereuse_devicehub.resources.lot import LotView from ereuse_devicehub.resources.lot.models import Lot -from ereuse_devicehub.resources.hash_reports import insert_hash - +from ereuse_devicehub.resources.hash_reports import insert_hash, ReportHash class Format(enum.Enum): @@ -193,6 +192,19 @@ class StockDocumentView(DeviceView): return output +class CheckView(View): + model = ReportHash + + def get(self): + qry = dict(request.values) + hash3 = qry['hash'] + + result = False + if ReportHash.query.filter_by(hash3=hash3).count(): + result = True + return jsonify(result) + + class DocumentDef(Resource): __type__ = 'Document' SCHEMA = None @@ -241,3 +253,6 @@ class DocumentDef(Resource): stock_view = StockDocumentView.as_view('stockDocumentView', definition=self, auth=app.auth) stock_view = app.auth.requires_auth(stock_view) self.add_url_rule('/stock/', defaults=d, view_func=stock_view, methods=get) + + check_view = CheckView.as_view('CheckView', definition=self, auth=app.auth) + self.add_url_rule('/check/', defaults={}, view_func=check_view, methods=get) diff --git a/tests/test_documents.py b/tests/test_documents.py index 162ed5a4..895aaf30 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -144,6 +144,9 @@ def test_check_insert_hash(app: Devicehub, user: UserClient): query=[('filter', {'type': ['Computer']})]) hash3 = hashlib.sha3_256(csv_str.encode('utf-8')).hexdigest() assert ReportHash.query.filter_by(hash3=hash3).count() == 1 + result, status = user.get(res=documents.DocumentDef.t, item='check/', query=[('hash', hash3)]) + assert status.status_code == 200 + assert result == True @pytest.mark.mvp From d89dbe8c2969507799c3753c8e4b449b138b868f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Dec 2020 13:53:48 +0100 Subject: [PATCH 7/8] fixing tests --- tests/test_basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_basic.py b/tests/test_basic.py index 8c62a216..ea8bac0d 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -53,6 +53,7 @@ def test_api_docs(client: Client): '/documents/lots/', '/documents/static/{filename}', '/documents/stock/', + '/documents/check/', '/drills/{dev1_id}/merge/{dev2_id}', '/graphic-cards/{dev1_id}/merge/{dev2_id}', '/hard-drives/{dev1_id}/merge/{dev2_id}', From 96540d701c6cd27b2895787a50ce69931b8091be Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 21 Dec 2020 16:09:30 +0100 Subject: [PATCH 8/8] fixing lineterminator in csv --- ereuse_devicehub/resources/documents/documents.py | 8 ++++---- tests/test_documents.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index fb5572bd..a7db64c6 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -117,7 +117,7 @@ class DevicesDocumentView(DeviceView): def generate_post_csv(self, query): """Get device query and put information in csv format.""" data = StringIO() - cw = csv.writer(data, delimiter=';', quotechar='"') + cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') first = True for device in query: d = DeviceRow(device) @@ -126,8 +126,8 @@ class DevicesDocumentView(DeviceView): first = False cw.writerow(d.values()) bfile = data.getvalue().encode('utf-8') - insert_hash(bfile) output = make_response(bfile) + insert_hash(bfile) output.headers['Content-Disposition'] = 'attachment; filename=export.csv' output.headers['Content-type'] = 'text/csv' return output @@ -197,10 +197,10 @@ class CheckView(View): def get(self): qry = dict(request.values) - hash3 = qry['hash'] + hash3 = qry.get('hash') result = False - if ReportHash.query.filter_by(hash3=hash3).count(): + if hash3 and ReportHash.query.filter_by(hash3=hash3).count(): result = True return jsonify(result) diff --git a/tests/test_documents.py b/tests/test_documents.py index 895aaf30..eecee5ad 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -135,7 +135,7 @@ def test_export_basic_snapshot(user: UserClient): @pytest.mark.mvp @pytest.mark.usefixtures(conftest.app_context.__name__) -def test_check_insert_hash(app: Devicehub, user: UserClient): +def test_check_insert_hash(app: Devicehub, user: UserClient, client: Client): """Test export device information in a csv file.""" snapshot, _ = user.post(file('basic.snapshot'), res=Snapshot) csv_str, _ = user.get(res=documents.DocumentDef.t, @@ -144,10 +144,17 @@ def test_check_insert_hash(app: Devicehub, user: UserClient): query=[('filter', {'type': ['Computer']})]) hash3 = hashlib.sha3_256(csv_str.encode('utf-8')).hexdigest() assert ReportHash.query.filter_by(hash3=hash3).count() == 1 - result, status = user.get(res=documents.DocumentDef.t, item='check/', query=[('hash', hash3)]) + result, status = client.get(res=documents.DocumentDef.t, item='check/', query=[('hash', hash3)]) assert status.status_code == 200 assert result == True + ff = open('/tmp/test.csv', 'w') + ff.write(csv_str) + ff.close() + + a= open('/tmp/test.csv').read() + assert hash3 == hashlib.sha3_256(a.encode('utf-8')).hexdigest() + @pytest.mark.mvp def test_export_extended(app: Devicehub, user: UserClient):