From ce4dce055d50cd159eef03ffb95dd2d33dfa2866 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 19 Feb 2021 12:53:42 +0100 Subject: [PATCH 01/11] basic endpoint for internal stats --- ereuse_devicehub/resources/documents/documents.py | 10 ++++++++++ tests/test_basic.py | 1 + tests/test_documents.py | 15 +++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index 33eca9b8..cd1d5254 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -252,6 +252,13 @@ class StampsView(View): return flask.render_template('documents/stamp.html', rq_url=url_path) +class InternalStatsView(View): + + def get(self): + result = '' + return jsonify(result) + + class DocumentDef(Resource): __type__ = 'Document' SCHEMA = None @@ -307,6 +314,9 @@ class DocumentDef(Resource): stamps_view = StampsView.as_view('StampsView', definition=self, auth=app.auth) self.add_url_rule('/stamps/', defaults={}, view_func=stamps_view, methods=get) + stamps_view = InternalStatsView.as_view('InternalStatsView', definition=self, auth=app.auth) + self.add_url_rule('/internalstats/', defaults={}, view_func=stamps_view, methods=get) + actions_view = ActionsDocumentView.as_view('ActionsDocumentView', definition=self, auth=app.auth) diff --git a/tests/test_basic.py b/tests/test_basic.py index b5e52805..d3b7e6e2 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -41,6 +41,7 @@ def test_api_docs(client: Client): '/documents/erasures/', '/documents/devices/', '/documents/stamps/', + '/documents/internalstats/', '/documents/stock/', '/documents/check/', '/documents/lots/', diff --git a/tests/test_documents.py b/tests/test_documents.py index fc4f8ea2..358ca869 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -459,3 +459,18 @@ def test_get_document_lots(user: UserClient, user2: UserClient): assert export_csv[1][3] == 'comments,lot1,testcomment-lot1,' or 'comments,lot2,testcomment-lot2,' assert export2_csv[1][1] == 'Lot3-User2' assert export2_csv[1][3] == 'comments,lot3,testcomment-lot3,' + + +@pytest.mark.mvp +def test_get_document_internal_stats(user: UserClient): + """Tests for get teh internal stats.""" + + # import pdb; pdb.set_trace() + csv_str, _ = user.get(res=documents.DocumentDef.t, + item='internalstats/') + + f = StringIO(csv_str) + obj_csv = csv.reader(f, f) + export_csv = list(obj_csv) + + assert len(export_csv) == 0 From da6cd547ed7a475668b328eb16526888e5f639d9 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 10:13:10 +0100 Subject: [PATCH 02/11] clean pdbs --- tests/test_documents.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_documents.py b/tests/test_documents.py index 358ca869..ecff4369 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -465,7 +465,6 @@ def test_get_document_lots(user: UserClient, user2: UserClient): def test_get_document_internal_stats(user: UserClient): """Tests for get teh internal stats.""" - # import pdb; pdb.set_trace() csv_str, _ = user.get(res=documents.DocumentDef.t, item='internalstats/') From d1b50cc212f0a74c40ad6eb7b21b27bcc3bf3fb5 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 15:49:13 +0100 Subject: [PATCH 03/11] simple internal stats --- .../resources/documents/device_row.py | 71 +++++++++++++++++++ .../resources/documents/documents.py | 39 ++++++++-- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/ereuse_devicehub/resources/documents/device_row.py b/ereuse_devicehub/resources/documents/device_row.py index 91c96000..fedef026 100644 --- a/ereuse_devicehub/resources/documents/device_row.py +++ b/ereuse_devicehub/resources/documents/device_row.py @@ -379,3 +379,74 @@ class ActionRow(OrderedDict): self['Type'] = allocate['type'] self['LiveCreate'] = allocate['liveCreate'] self['UsageTimeHdd'] = allocate['usageTimeHdd'] + + +class InternalStatsRow(OrderedDict): + + def __init__(self, user, create, actions): + super().__init__() + # General information about all internal stats + # user, quart, month, year: + # Snapshot (Registers) + # Snapshots (Update) + # Snapshots (All) + # Allocate + # Deallocate + # Live + self.actions = actions + year, month = create.split('-') + + self['User'] = user + self['Year'] = year + self['Quarter'] = self.quarter(month) + self['Month'] = month + self['Snapshot (Registers)'] = 0 + self['Snapshot (Update)'] = 0 + self['Snapshot (All)'] = 0 + self['Allocates'] = 0 + self['Deallocates'] = 0 + self['Lives'] = 0 + + self.count_actions() + + def count_actions(self): + for ac in self.actions: + self.is_snapshot( + self.is_deallocate( + self.is_allocate(ac) + ) + ) + + def is_allocate(self, ac): + if ac.type == 'Allocate': + self['Allocates'] += 1 + return ac + + def is_deallocate(self, ac): + if ac.type == 'Deallocate': + self['Deallocates'] += 1 + return ac + + def is_snapshot(self, ac): + if not ac.type == 'Snapshot': + return + self['Snapshot (All)'] += 1 + + canary = False + for _ac in ac.device.actions: + if _ac.created < ac.created: + canary = True + break + + if canary: + self['Snapshot (Update)'] += 1 + else: + self['Snapshot (Registers)'] += 1 + + def quarter(self, month): + q = {1: 'Q1', 2: 'Q1', 3: 'Q1', + 4: 'Q2', 5: 'Q2', 6: 'Q2', + 7: 'Q3', 8: 'Q3', 9: 'Q3', + 10: 'Q4', 11: 'Q4', 12: 'Q4', + } + return q[int(month)] diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index cd1d5254..d84021d2 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -21,7 +21,8 @@ from ereuse_devicehub.resources.action import models as evs from ereuse_devicehub.resources.device import models as devs from ereuse_devicehub.resources.deliverynote.models import Deliverynote from ereuse_devicehub.resources.device.views import DeviceView -from ereuse_devicehub.resources.documents.device_row import DeviceRow, StockRow, ActionRow +from ereuse_devicehub.resources.documents.device_row import (DeviceRow, StockRow, ActionRow, + InternalStatsRow) from ereuse_devicehub.resources.lot import LotView from ereuse_devicehub.resources.lot.models import Lot from ereuse_devicehub.resources.hash_reports import insert_hash, ReportHash @@ -212,7 +213,7 @@ class StockDocumentView(DeviceView): def generate_post_csv(self, query): """Get device query and put information in csv format.""" data = StringIO() - cw = csv.writer(data) + cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') first = True for device in query: d = StockRow(device) @@ -255,8 +256,38 @@ class StampsView(View): class InternalStatsView(View): def get(self): - result = '' - return jsonify(result) + return self.get_datas() + + def get_datas(self): + actions = evs.Action.query.filter( + evs.Action.type.in_(('Snapshot', 'Live', 'Allocate', 'Deallocate'))) + d = {} + for ac in actions: + create = '{}-{}'.format(ac.created.year, ac.created.month) + user = ac.author.email + + if not user in d: + d[user] = {} + + if not create in d[user]: + d[user][create] = [] + + d[user][create].append(ac) + + + data = StringIO() + cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') + cw.writerow(InternalStatsRow('', "2000-1", []).keys()) + for user, createds in d.items(): + for create, actions in createds.items(): + cw.writerow(InternalStatsRow(user, create, actions).values()) + + bfile = data.getvalue().encode('utf-8') + output = make_response(bfile) + insert_hash(bfile) + output.headers['Content-Disposition'] = 'attachment; filename=internal-stats.csv' + output.headers['Content-type'] = 'text/csv' + return output class DocumentDef(Resource): From 181b7da69f9ff7430a2d5379b2f07d07f3ec248b Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 21:14:56 +0100 Subject: [PATCH 04/11] adding EMAIL_ADMIN to config --- ereuse_devicehub/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ereuse_devicehub/config.py b/ereuse_devicehub/config.py index 449f104d..beef8b25 100644 --- a/ereuse_devicehub/config.py +++ b/ereuse_devicehub/config.py @@ -65,3 +65,6 @@ class DevicehubConfig(Config): PRICE_VERSION = StrictVersion('1.0') PRICE_CURRENCY = Currency.EUR """Official versions.""" + + """Admin email""" + EMAIL_ADMIN = config('EMAIL_ADMIN', '') From d956271908e09a17ed30ba5bf55e12894f1b373b Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 21:15:25 +0100 Subject: [PATCH 05/11] adding EMAIL_ADMIN to test config --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index d212a9b7..d0a0bdd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,7 @@ class TestConfig(DevicehubConfig): SERVER_NAME = 'localhost' TMP_SNAPSHOTS = '/tmp/snapshots' TMP_LIVES = '/tmp/lives' + EMAIL_ADMIN = 'foo@foo.com' @pytest.fixture(scope='session') From 30005e1f913ddd9abd4ca90e4b42419be9da5b90 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 21:16:32 +0100 Subject: [PATCH 06/11] remove pdbs --- tests/test_basic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_basic.py b/tests/test_basic.py index d3b7e6e2..11ffbb80 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -27,7 +27,6 @@ def test_dependencies(): def test_api_docs(client: Client): """Tests /apidocs correct initialization.""" docs, _ = client.get('/apidocs') - # import pdb; pdb.set_trace() assert set(docs['paths'].keys()) == { '/actions/', '/apidocs', From 7552839f9c2d7f6bedf7089b120ae9bcdf15a62e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 21:17:07 +0100 Subject: [PATCH 07/11] updating test for internal stats --- tests/test_documents.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/test_documents.py b/tests/test_documents.py index ecff4369..74698676 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -462,14 +462,29 @@ def test_get_document_lots(user: UserClient, user2: UserClient): @pytest.mark.mvp -def test_get_document_internal_stats(user: UserClient): +def test_get_document_internal_stats(user: UserClient, user2: UserClient): """Tests for get teh internal stats.""" + # csv_str, _ = user.get(res=documents.DocumentDef.t, + # item='internalstats/') csv_str, _ = user.get(res=documents.DocumentDef.t, - item='internalstats/') + item='internalstats/', + accept='text/csv', + query=[]) f = StringIO(csv_str) obj_csv = csv.reader(f, f) export_csv = list(obj_csv) - assert len(export_csv) == 0 + assert len(export_csv) == 1 + + csv_str, _ = user2.get(res=documents.DocumentDef.t, + item='internalstats/', + accept='text/csv', + query=[]) + + f = StringIO(csv_str) + obj_csv = csv.reader(f, f) + export_csv = list(obj_csv) + + assert csv_str.strip() == '""' From 2a38070ae07a3329abb713e99f6ac00db13b3b84 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 21:18:25 +0100 Subject: [PATCH 08/11] prepare auth for internal stats --- .../resources/documents/documents.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index d84021d2..692e6992 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -5,6 +5,7 @@ import uuid from collections import OrderedDict from io import StringIO from typing import Callable, Iterable, Tuple +from decouple import config import boltons import flask @@ -12,6 +13,7 @@ import flask_weasyprint import teal.marshmallow from boltons import urlutils from flask import make_response, g, request +from flask import current_app as app from flask.json import jsonify from teal.cache import cache from teal.resource import Resource, View @@ -253,28 +255,28 @@ class StampsView(View): return flask.render_template('documents/stamp.html', rq_url=url_path) -class InternalStatsView(View): - - def get(self): - return self.get_datas() - - def get_datas(self): - actions = evs.Action.query.filter( +class InternalStatsView(DeviceView): + @cache(datetime.timedelta(minutes=1)) + def find(self, args: dict): + if not g.user.email == app.config['EMAIL_ADMIN']: + return jsonify('') + query = evs.Action.query.filter( evs.Action.type.in_(('Snapshot', 'Live', 'Allocate', 'Deallocate'))) + return self.generate_post_csv(query) + + + def generate_post_csv(self, query): d = {} - for ac in actions: + for ac in query: create = '{}-{}'.format(ac.created.year, ac.created.month) user = ac.author.email if not user in d: - d[user] = {} - + d[user] = {} if not create in d[user]: d[user][create] = [] - d[user][create].append(ac) - data = StringIO() cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') cw.writerow(InternalStatsRow('', "2000-1", []).keys()) @@ -345,11 +347,14 @@ class DocumentDef(Resource): stamps_view = StampsView.as_view('StampsView', definition=self, auth=app.auth) self.add_url_rule('/stamps/', defaults={}, view_func=stamps_view, methods=get) - stamps_view = InternalStatsView.as_view('InternalStatsView', definition=self, auth=app.auth) - self.add_url_rule('/internalstats/', defaults={}, view_func=stamps_view, methods=get) + internalstats_view = InternalStatsView.as_view( + 'InternalStatsView', definition=self, auth=app.auth) + internalstats_view = app.auth.requires_auth(internalstats_view) + self.add_url_rule('/internalstats/', defaults=d, view_func=internalstats_view, + methods=get) - actions_view = ActionsDocumentView.as_view('ActionsDocumentView', - definition=self, + actions_view = ActionsDocumentView.as_view('ActionsDocumentView', + definition=self, auth=app.auth) actions_view = app.auth.requires_auth(actions_view) self.add_url_rule('/actions/', defaults=d, view_func=actions_view, methods=get) From e85b40c90f5689b5a61626c64e6389765def9836 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 22 Feb 2021 22:08:14 +0100 Subject: [PATCH 09/11] Fixing test stock --- tests/test_documents.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_documents.py b/tests/test_documents.py index 74698676..5ac747b5 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -414,6 +414,8 @@ def test_report_devices_stock_control(user: UserClient, user2: UserClient): assert user.user['id'] != user2.user['id'] assert len(export_csv) == 2 + export_csv[0] = export_csv[0][0].split(';') + export_csv[1] = export_csv[1][0].split(';') assert isinstance(datetime.strptime(export_csv[1][5], '%c'), datetime), \ 'Register in field is not a datetime' From 79902ee26d8f41e9ed98f69bd7ecdd22543835b5 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 23 Feb 2021 12:23:07 +0100 Subject: [PATCH 10/11] adding lives action to the document of internal stats --- CHANGELOG.md | 1 + ereuse_devicehub/resources/documents/device_row.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 391163d2..141fb297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ml). [1.0.5-beta] ## [1.0.5-beta] +- [addend] #124 adding endpoint for extract the internal stats of use ## [1.0.4-beta] - [addend] #95 adding endpoint for check the hash of one report diff --git a/ereuse_devicehub/resources/documents/device_row.py b/ereuse_devicehub/resources/documents/device_row.py index fedef026..fd37760f 100644 --- a/ereuse_devicehub/resources/documents/device_row.py +++ b/ereuse_devicehub/resources/documents/device_row.py @@ -413,7 +413,9 @@ class InternalStatsRow(OrderedDict): for ac in self.actions: self.is_snapshot( self.is_deallocate( - self.is_allocate(ac) + self.is_live( + self.is_allocate(ac) + ) ) ) @@ -422,6 +424,11 @@ class InternalStatsRow(OrderedDict): self['Allocates'] += 1 return ac + def is_live(self, ac): + if ac.type == 'Live': + self['Lives'] += 1 + return ac + def is_deallocate(self, ac): if ac.type == 'Deallocate': self['Deallocates'] += 1 From 4c08f65218611471aa6a77bb69d2024d86c6f42e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Tue, 23 Feb 2021 17:40:36 +0100 Subject: [PATCH 11/11] fixing internal stats --- ereuse_devicehub/resources/documents/device_row.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ereuse_devicehub/resources/documents/device_row.py b/ereuse_devicehub/resources/documents/device_row.py index fd37760f..74a322be 100644 --- a/ereuse_devicehub/resources/documents/device_row.py +++ b/ereuse_devicehub/resources/documents/device_row.py @@ -441,6 +441,9 @@ class InternalStatsRow(OrderedDict): canary = False for _ac in ac.device.actions: + if not _ac.type == 'Snapshot': + continue + if _ac.created < ac.created: canary = True break