From 1cb0634bf167ca7442e4de6f2bb0c57482f04b96 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 2 Nov 2022 09:57:50 +0100 Subject: [PATCH 1/4] add ignore files --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 0d6f75c3..5b2c87d1 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,8 @@ yarn.lock # ESLint Report eslint_report.json +modules/ +tmp/ +.env* +bin/ +env* From eae036d7c0f122ee7588c0c4b37725103d056e90 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 2 Nov 2022 12:00:39 +0100 Subject: [PATCH 2/4] remove internal stats of actual devicehub --- ereuse_devicehub/commands/reports.py | 96 ----------------- ereuse_devicehub/devicehub.py | 6 +- .../resources/documents/device_row.py | 101 ------------------ .../resources/documents/documents.py | 43 -------- 4 files changed, 3 insertions(+), 243 deletions(-) delete mode 100644 ereuse_devicehub/commands/reports.py diff --git a/ereuse_devicehub/commands/reports.py b/ereuse_devicehub/commands/reports.py deleted file mode 100644 index ae8bd501..00000000 --- a/ereuse_devicehub/commands/reports.py +++ /dev/null @@ -1,96 +0,0 @@ -import csv - -# import click_spinner -# import ereuse_utils.cli -from io import StringIO - -from ereuse_devicehub.resources.action import models as evs -from ereuse_devicehub.resources.device.models import Placeholder -from ereuse_devicehub.resources.documents.device_row import InternalStatsRow - -# import click - - -class Report: - def __init__(self, app) -> None: - super().__init__() - self.app = app - short_help = 'Creates reports devices and users.' - self.app.cli.command('report', short_help=short_help)(self.run) - - def run(self): - stats = InternalStatsView() - stats.print() - - -class InternalStatsView: - def print(self): - query = evs.Action.query.filter( - evs.Action.type.in_( - ( - 'Snapshot', - 'Live', - 'Allocate', - 'Deallocate', - 'EraseBasic', - 'EraseSectors', - ) - ) - ) - return self.generate_post_csv(query) - - def generate_post_csv(self, query): - data = StringIO() - cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') - cw.writerow(InternalStatsRow('', "2000-1", [], []).keys()) - - for row in self.get_rows(query): - cw.writerow(row) - - return print(data.getvalue()) - - def get_rows(self, query): - d = {} - dd = {} - disks = [] - for ac in query: - create = '{}-{}'.format(ac.created.year, ac.created.month) - user = ac.author.email - - if user not in d: - d[user] = {} - dd[user] = {} - if create not in d[user]: - d[user][create] = [] - dd[user][create] = None - d[user][create].append(ac) - - for user, createds in d.items(): - for create, actions in createds.items(): - r = InternalStatsRow(user, create, actions, disks) - dd[user][create] = r - - return self.get_placeholders(dd) - - def get_placeholders(self, dd): - - for p in Placeholder.query.all(): - create = '{}-{}'.format(p.created.year, p.created.month) - user = p.owner.email - - if user not in dd: - dd[user] = {} - - if create not in dd[user]: - dd[user][create] = None - - if not dd[user][create]: - dd[user][create] = InternalStatsRow(user, create, [], []) - - dd[user][create]['Placeholders'] += 1 - - rows = [] - for user, createds in dd.items(): - for create, row in createds.items(): - rows.append(row.values()) - return rows diff --git a/ereuse_devicehub/devicehub.py b/ereuse_devicehub/devicehub.py index 5fec2cbe..c82f2fd1 100644 --- a/ereuse_devicehub/devicehub.py +++ b/ereuse_devicehub/devicehub.py @@ -15,7 +15,7 @@ from teal.teal import Teal from ereuse_devicehub.auth import Auth from ereuse_devicehub.client import Client, UserClient -from ereuse_devicehub.commands.reports import Report +# from ereuse_devicehub.commands.reports import Report from ereuse_devicehub.commands.users import GetToken from ereuse_devicehub.config import DevicehubConfig from ereuse_devicehub.db import db @@ -29,7 +29,7 @@ from ereuse_devicehub.templating import Environment class Devicehub(Teal): test_client_class = Client Dummy = Dummy - Report = Report + # Report = Report jinja_environment = Environment def __init__( @@ -70,7 +70,7 @@ class Devicehub(Teal): self.id = inventory """The Inventory ID of this instance. In Teal is the app.schema.""" self.dummy = Dummy(self) - self.report = Report(self) + # self.report = Report(self) self.get_token = GetToken(self) @self.cli.group( diff --git a/ereuse_devicehub/resources/documents/device_row.py b/ereuse_devicehub/resources/documents/device_row.py index b29c60b4..2e13d327 100644 --- a/ereuse_devicehub/resources/documents/device_row.py +++ b/ereuse_devicehub/resources/documents/device_row.py @@ -614,104 +614,3 @@ class ActionRow(OrderedDict): self['Type'] = allocate['type'] self['LiveCreate'] = allocate['liveCreate'] self['UsageTimeHdd'] = allocate['usageTimeHdd'] - - -class InternalStatsRow(OrderedDict): - def __init__(self, user, create, actions, disks): - super().__init__() - # General information about all internal stats - # user, quart, month, year: - # Snapshot (Registers) - # Snapshots (Update) - # Snapshots (All) - # Drives Erasure - # Drives Erasure Uniques - # Placeholders - # Allocate - # Deallocate - # Live - self.actions = actions - year, month = create.split('-') - self.disks = disks - - self['User'] = user - self['Year'] = year - self['Quarter'] = self.quarter(month) - self['Month'] = month - self['Snapshot (Registers)'] = 0 - self['Snapshot (Update)'] = 0 - self['Snapshot (All)'] = 0 - self['Drives Erasure'] = 0 - self['Drives Erasure Uniques'] = 0 - self['Placeholders'] = 0 - self['Allocates'] = 0 - self['Deallocates'] = 0 - self['Lives'] = 0 - - self.count_actions() - - def count_actions(self): - for ac in self.actions: - self.is_snapshot( - self.is_deallocate(self.is_live(self.is_allocate(self.is_erase(ac)))) - ) - - def is_allocate(self, ac): - if ac.type == 'Allocate': - self['Allocates'] += 1 - return ac - - def is_erase(self, ac): - if ac.type in ['EraseBasic', 'EraseSectors']: - self['Drives Erasure'] += 1 - if ac.device in self.disks: - return ac - self['Drives Erasure Uniques'] += 1 - self.disks.append(ac.device) - return ac - - def is_live(self, ac): - if ac.type == 'Live': - self['Lives'] += 1 - return ac - - def is_deallocate(self, ac): - if ac.type == 'Deallocate': - self['Deallocates'] += 1 - return ac - - def is_snapshot(self, ac): - if not ac.type == 'Snapshot': - return - self['Snapshot (All)'] += 1 - - canary = False - for _ac in ac.device.actions: - if not _ac.type == 'Snapshot': - continue - - if _ac.created < ac.created: - canary = True - break - - if canary: - self['Snapshot (Update)'] += 1 - else: - self['Snapshot (Registers)'] += 1 - - def quarter(self, month): - q = { - 1: 'Q1', - 2: 'Q1', - 3: 'Q1', - 4: 'Q2', - 5: 'Q2', - 6: 'Q2', - 7: 'Q3', - 8: 'Q3', - 9: 'Q3', - 10: 'Q4', - 11: 'Q4', - 12: 'Q4', - } - return q[int(month)] diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index ca66308f..37ec5914 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -30,7 +30,6 @@ from ereuse_devicehub.resources.device.views import DeviceView from ereuse_devicehub.resources.documents.device_row import ( ActionRow, DeviceRow, - InternalStatsRow, StockRow, ) from ereuse_devicehub.resources.enums import SessionType @@ -345,42 +344,6 @@ class StampsView(View): result=result) -class InternalStatsView(DeviceView): - @cache(datetime.timedelta(minutes=1)) - def find(self, args: dict): - if not g.user.email == app.config['EMAIL_ADMIN']: - return jsonify('') - query = evs.Action.query.filter( - evs.Action.type.in_(('Snapshot', 'Live', 'Allocate', 'Deallocate'))) - return self.generate_post_csv(query) - - def generate_post_csv(self, query): - d = {} - for ac in query: - create = '{}-{}'.format(ac.created.year, ac.created.month) - user = ac.author.email - - if user not in d: - d[user] = {} - if create not in d[user]: - d[user][create] = [] - d[user][create].append(ac) - - data = StringIO() - cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') - cw.writerow(InternalStatsRow('', "2000-1", []).keys()) - for user, createds in d.items(): - for create, actions in createds.items(): - cw.writerow(InternalStatsRow(user, create, actions).values()) - - bfile = data.getvalue().encode('utf-8') - output = make_response(bfile) - insert_hash(bfile) - output.headers['Content-Disposition'] = 'attachment; filename=internal-stats.csv' - output.headers['Content-type'] = 'text/csv' - return output - - class WbConfDocumentView(DeviceView): def get(self, wbtype: str): if not wbtype.lower() in ['usodyrate', 'usodywipe']: @@ -473,12 +436,6 @@ class DocumentDef(Resource): stamps_view = StampsView.as_view('StampsView', definition=self, auth=app.auth) self.add_url_rule('/stamps/', defaults={}, view_func=stamps_view, methods={'GET', 'POST'}) - # internalstats_view = InternalStatsView.as_view( - # 'InternalStatsView', definition=self, auth=app.auth) - # internalstats_view = app.auth.requires_auth(internalstats_view) - # self.add_url_rule('/internalstats/', defaults=d, view_func=internalstats_view, - # methods=get) - actions_view = ActionsDocumentView.as_view('ActionsDocumentView', definition=self, auth=app.auth) From 6c7ae1bb757dd338e4f61455338984f79ca3acc5 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 2 Nov 2022 19:44:39 +0100 Subject: [PATCH 3/4] remove report of devicehub commands --- ereuse_devicehub/devicehub.py | 1 + .../resources/documents/documents.py | 152 ++++++++++++------ 2 files changed, 104 insertions(+), 49 deletions(-) diff --git a/ereuse_devicehub/devicehub.py b/ereuse_devicehub/devicehub.py index c82f2fd1..82f5dd4a 100644 --- a/ereuse_devicehub/devicehub.py +++ b/ereuse_devicehub/devicehub.py @@ -15,6 +15,7 @@ from teal.teal import Teal from ereuse_devicehub.auth import Auth from ereuse_devicehub.client import Client, UserClient + # from ereuse_devicehub.commands.reports import Report from ereuse_devicehub.commands.users import GetToken from ereuse_devicehub.config import DevicehubConfig diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index 37ec5914..d33fd238 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -61,9 +61,9 @@ class DocumentView(DeviceView): 200: description: Return the collection or the specific one. """ - args = self.QUERY_PARSER.parse(self.find_args, - flask.request, - locations=('querystring',)) + args = self.QUERY_PARSER.parse( + self.find_args, flask.request, locations=('querystring',) + ) ids = [] if 'filter' in request.args: filters = json.loads(request.args.get('filter', {})) @@ -112,7 +112,8 @@ class DocumentView(DeviceView): template = self.erasure(query) if args.get('format') == Format.PDF: res = flask_weasyprint.render_pdf( - flask_weasyprint.HTML(string=template), download_filename='{}.pdf'.format(type) + flask_weasyprint.HTML(string=template), + download_filename='{}.pdf'.format(type), ) insert_hash(res.data) else: @@ -139,7 +140,7 @@ class DocumentView(DeviceView): params = { 'title': 'Erasure Certificate', 'erasures': tuple(erasures()), - 'url_pdf': url_pdf.to_text() + 'url_pdf': url_pdf.to_text(), } return flask.render_template('documents/erasure.html', **params) @@ -158,7 +159,13 @@ class DevicesDocumentView(DeviceView): def generate_post_csv(self, query): """Get device query and put information in csv format.""" data = StringIO() - cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL) + cw = csv.writer( + data, + delimiter=';', + lineterminator="\n", + quotechar='"', + quoting=csv.QUOTE_ALL, + ) first = True document_ids = self.get_documents_id() for device in query: @@ -192,7 +199,13 @@ class ActionsDocumentView(DeviceView): def generate_post_csv(self, query): """Get device query and put information in csv format.""" data = StringIO() - cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL) + cw = csv.writer( + data, + delimiter=';', + lineterminator="\n", + quotechar='"', + quoting=csv.QUOTE_ALL, + ) first = True devs_id = [] for device in query: @@ -203,7 +216,9 @@ class ActionsDocumentView(DeviceView): cw.writerow(d.keys()) first = False cw.writerow(d.values()) - query_trade = Trade.query.filter(Trade.devices.any(Device.id.in_(devs_id))).all() + query_trade = Trade.query.filter( + Trade.devices.any(Device.id.in_(devs_id)) + ).all() lot_id = request.args.get('lot') if lot_id and not query_trade: @@ -224,7 +239,9 @@ class ActionsDocumentView(DeviceView): bfile = data.getvalue().encode('utf-8') output = make_response(bfile) insert_hash(bfile) - output.headers['Content-Disposition'] = 'attachment; filename=actions_export.csv' + output.headers[ + 'Content-Disposition' + ] = 'attachment; filename=actions_export.csv' output.headers['Content-type'] = 'text/csv' return output @@ -276,7 +293,13 @@ class StockDocumentView(DeviceView): def generate_post_csv(self, query): """Get device query and put information in csv format.""" data = StringIO() - cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL) + cw = csv.writer( + data, + delimiter=';', + lineterminator="\n", + quotechar='"', + quoting=csv.QUOTE_ALL, + ) first = True for device in query: d = StockRow(device) @@ -310,6 +333,7 @@ class StampsView(View): This view render one public ans static page for see the links for to do the check of one csv file """ + def get_url_path(self): url = urlutils.URL(request.url) url.normalize() @@ -318,8 +342,9 @@ class StampsView(View): def get(self): result = ('', '') - return flask.render_template('documents/stamp.html', rq_url=self.get_url_path(), - result=result) + return flask.render_template( + 'documents/stamp.html', rq_url=self.get_url_path(), result=result + ) def post(self): result = ('', '') @@ -330,18 +355,26 @@ class StampsView(View): ok = '100% coincidence. The attached file contains data 100% existing in \ to our backend' result = ('Bad', bad) - mime = ['text/csv', 'application/pdf', 'text/plain', 'text/markdown', - 'image/jpeg', 'image/png', 'text/html', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.oasis.opendocument.spreadsheet', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/msword'] + mime = [ + 'text/csv', + 'application/pdf', + 'text/plain', + 'text/markdown', + 'image/jpeg', + 'image/png', + 'text/html', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.oasis.opendocument.spreadsheet', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/msword', + ] if file_check.mimetype in mime: if verify_hash(file_check): result = ('Ok', ok) - return flask.render_template('documents/stamp.html', rq_url=self.get_url_path(), - result=result) + return flask.render_template( + 'documents/stamp.html', rq_url=self.get_url_path(), result=result + ) class WbConfDocumentView(DeviceView): @@ -349,10 +382,11 @@ class WbConfDocumentView(DeviceView): if not wbtype.lower() in ['usodyrate', 'usodywipe']: return jsonify('') - data = {'token': self.get_token(), - 'host': app.config['HOST'], - 'inventory': app.config['SCHEMA'] - } + data = { + 'token': self.get_token(), + 'host': app.config['HOST'], + 'inventory': app.config['SCHEMA'], + } data['erase'] = False # data['erase'] = True if wbtype == 'usodywipe' else False @@ -387,18 +421,31 @@ class DocumentDef(Resource): VIEW = None # We do not want to create default / documents endpoint AUTH = False - def __init__(self, app, - import_name=__name__, - static_folder='static', - static_url_path=None, - template_folder='templates', - url_prefix=None, - subdomain=None, - url_defaults=None, - root_path=None, - cli_commands: Iterable[Tuple[Callable, str or None]] = tuple()): - super().__init__(app, import_name, static_folder, static_url_path, template_folder, - url_prefix, subdomain, url_defaults, root_path, cli_commands) + def __init__( + self, + app, + import_name=__name__, + static_folder='static', + static_url_path=None, + template_folder='templates', + url_prefix=None, + subdomain=None, + url_defaults=None, + root_path=None, + cli_commands: Iterable[Tuple[Callable, str or None]] = tuple(), + ): + super().__init__( + app, + import_name, + static_folder, + static_url_path, + template_folder, + url_prefix, + subdomain, + url_defaults, + root_path, + cli_commands, + ) d = {'id': None} get = {'GET'} @@ -409,12 +456,15 @@ class DocumentDef(Resource): view = app.auth.requires_auth(view) self.add_url_rule('/erasures/', defaults=d, view_func=view, methods=get) - self.add_url_rule('/erasures/<{}:{}>'.format(self.ID_CONVERTER.value, self.ID_NAME), - view_func=view, methods=get) + self.add_url_rule( + '/erasures/<{}:{}>'.format(self.ID_CONVERTER.value, self.ID_NAME), + view_func=view, + methods=get, + ) - devices_view = DevicesDocumentView.as_view('devicesDocumentView', - definition=self, - auth=app.auth) + devices_view = DevicesDocumentView.as_view( + 'devicesDocumentView', definition=self, auth=app.auth + ) devices_view = app.auth.requires_auth(devices_view) stock_view = StockDocumentView.as_view('stockDocumentView', definition=self) @@ -426,7 +476,9 @@ class DocumentDef(Resource): lots_view = app.auth.requires_auth(lots_view) self.add_url_rule('/lots/', defaults=d, view_func=lots_view, methods=get) - stock_view = StockDocumentView.as_view('stockDocumentView', definition=self, auth=app.auth) + stock_view = StockDocumentView.as_view( + 'stockDocumentView', definition=self, auth=app.auth + ) stock_view = app.auth.requires_auth(stock_view) self.add_url_rule('/stock/', defaults=d, view_func=stock_view, methods=get) @@ -434,16 +486,18 @@ class DocumentDef(Resource): self.add_url_rule('/check/', defaults={}, view_func=check_view, methods=get) stamps_view = StampsView.as_view('StampsView', definition=self, auth=app.auth) - self.add_url_rule('/stamps/', defaults={}, view_func=stamps_view, methods={'GET', 'POST'}) + self.add_url_rule( + '/stamps/', defaults={}, view_func=stamps_view, methods={'GET', 'POST'} + ) - actions_view = ActionsDocumentView.as_view('ActionsDocumentView', - definition=self, - auth=app.auth) + actions_view = ActionsDocumentView.as_view( + 'ActionsDocumentView', definition=self, auth=app.auth + ) actions_view = app.auth.requires_auth(actions_view) self.add_url_rule('/actions/', defaults=d, view_func=actions_view, methods=get) - wbconf_view = WbConfDocumentView.as_view('WbConfDocumentView', - definition=self, - auth=app.auth) + wbconf_view = WbConfDocumentView.as_view( + 'WbConfDocumentView', definition=self, auth=app.auth + ) wbconf_view = app.auth.requires_auth(wbconf_view) self.add_url_rule('/wbconf/', view_func=wbconf_view, methods=get) From 6ee62eb41528dbc272c48fa7dd8783ea4d993f30 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 3 Nov 2022 13:28:49 +0100 Subject: [PATCH 4/4] test --- tests/test_render_2_0.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_render_2_0.py b/tests/test_render_2_0.py index f95adf40..9318d3b7 100644 --- a/tests/test_render_2_0.py +++ b/tests/test_render_2_0.py @@ -1,10 +1,11 @@ import datetime +import pytest import json + from io import BytesIO from pathlib import Path from uuid import UUID -import pytest from flask import g from flask.testing import FlaskClient from flask_wtf.csrf import generate_csrf