From af885c2aec428ba634f7d0e5bcf824a0e07e7459 Mon Sep 17 00:00:00 2001 From: Xavier Bustamante Talavera Date: Mon, 2 Jul 2018 12:52:54 +0200 Subject: [PATCH] First real snapshots :-) --- README.md | 15 +- ereuse_devicehub/config.py | 7 +- ereuse_devicehub/dummy/dummy.py | 27 ++- .../dummy/files/real-hp.snapshot.11.yaml | 160 ++++++++++++++++++ .../dummy/files/real-toshiba.snapshot.11.yaml | 139 +++++++++++++++ .../files/workbench-server-1.snapshot.yaml | 11 +- ereuse_devicehub/resources/device/__init__.py | 7 +- ereuse_devicehub/resources/device/models.py | 26 +-- ereuse_devicehub/resources/device/schemas.py | 20 ++- ereuse_devicehub/resources/enums.py | 2 + ereuse_devicehub/resources/event/models.py | 47 ++--- ereuse_devicehub/resources/event/models.pyi | 5 +- ereuse_devicehub/resources/event/schemas.py | 11 +- setup.py | 6 +- tests/files/erase-sectors.snapshot.yaml | 7 +- tests/files/real-hp.snapshot.11.yaml | 160 ++++++++++++++++++ tests/files/real-toshiba.snapshot.11.yaml | 139 +++++++++++++++ tests/files/workbench-server-1.snapshot.yaml | 1 - tests/files/workbench-server-3.erase.yaml | 5 +- tests/test_basic.py | 2 +- tests/test_event.py | 33 ++-- tests/test_snapshot.py | 2 - tests/test_workbench.py | 21 +++ 23 files changed, 732 insertions(+), 121 deletions(-) create mode 100644 ereuse_devicehub/dummy/files/real-hp.snapshot.11.yaml create mode 100644 ereuse_devicehub/dummy/files/real-toshiba.snapshot.11.yaml create mode 100644 tests/files/real-hp.snapshot.11.yaml create mode 100644 tests/files/real-toshiba.snapshot.11.yaml diff --git a/README.md b/README.md index 586d9dbf..c74f0f70 100644 --- a/README.md +++ b/README.md @@ -75,4 +75,17 @@ Devicehub has many commands that allows you to administrate it. You can, for example, create a dummy database of devices with ``flask dummy`` or create users with ``flask create-user``. See all the available commands by just executing ``flask`` and get more information -per command by executing ``flask command --help``. \ No newline at end of file +per command by executing ``flask command --help``. + +## Understand the software +See the [docs](docs/index.rst) to understand how the software works and +the design principles. + +### Use the API +Checkout [Swagger](https://app.swaggerhub.com/apis/ereuse/devicehub/0.2) +to see the schemas and endpoints (we are working in making it +interactive). + +Use postman as an example of how to use the API. + +[![Run in Postman](https://run.pstmn.io/button.svg)](https://documenter.getpostman.com/view/254251/RWEnmFPs) diff --git a/ereuse_devicehub/config.py b/ereuse_devicehub/config.py index 2cd0474e..9e3845d6 100644 --- a/ereuse_devicehub/config.py +++ b/ereuse_devicehub/config.py @@ -5,7 +5,7 @@ from ereuse_devicehub.resources.device import CellphoneDef, ComponentDef, Comput ComputerMonitorDef, DataStorageDef, DesktopDef, DeviceDef, DisplayDef, GraphicCardDef, \ HardDriveDef, LaptopDef, MobileDef, MonitorDef, MotherboardDef, NetworkAdapterDef, \ ProcessorDef, RamModuleDef, ServerDef, SmartphoneDef, SolidStateDriveDef, TabletDef, \ - TelevisionSetDef + TelevisionSetDef, SoundCardDef from ereuse_devicehub.resources.event import AddDef, AggregateRateDef, AppRateDef, \ BenchmarkDataStorageDef, BenchmarkDef, BenchmarkProcessorDef, BenchmarkProcessorSysbenchDef, \ BenchmarkRamSysbenchDef, BenchmarkWithRateDef, EraseBasicDef, EraseSectorsDef, EventDef, \ @@ -23,7 +23,8 @@ class DevicehubConfig(Config): DeviceDef, ComputerDef, DesktopDef, LaptopDef, ServerDef, MonitorDef, TelevisionSetDef, ComputerMonitorDef, ComponentDef, GraphicCardDef, DataStorageDef, SolidStateDriveDef, MobileDef, DisplayDef, SmartphoneDef, TabletDef, CellphoneDef, - HardDriveDef, MotherboardDef, NetworkAdapterDef, RamModuleDef, ProcessorDef, UserDef, + HardDriveDef, MotherboardDef, NetworkAdapterDef, RamModuleDef, ProcessorDef, SoundCardDef, + UserDef, OrganizationDef, TagDef, EventDef, AddDef, RemoveDef, EraseBasicDef, EraseSectorsDef, StepDef, StepZeroDef, StepRandomDef, RateDef, AggregateRateDef, WorkbenchRateDef, PhotoboxUserDef, PhotoboxSystemRateDef, InstallDef, SnapshotDef, TestDef, @@ -33,7 +34,7 @@ class DevicehubConfig(Config): } PASSWORD_SCHEMES = {'pbkdf2_sha256'} # type: Set[str] SQLALCHEMY_DATABASE_URI = 'postgresql://dhub:ereuse@localhost/devicehub' # type: str - MIN_WORKBENCH = StrictVersion('11.0') # type: StrictVersion + MIN_WORKBENCH = StrictVersion('11.0a1') # type: StrictVersion """ the minimum algorithm_version of ereuse.org workbench that this devicehub accepts. we recommend not changing this value. diff --git a/ereuse_devicehub/dummy/dummy.py b/ereuse_devicehub/dummy/dummy.py index 0963d00e..a1307410 100644 --- a/ereuse_devicehub/dummy/dummy.py +++ b/ereuse_devicehub/dummy/dummy.py @@ -3,11 +3,10 @@ from pathlib import Path import click import click_spinner import yaml -from tqdm import tqdm - from ereuse_devicehub.client import UserClient from ereuse_devicehub.db import db from ereuse_devicehub.resources.event.models import Snapshot +from ereuse_devicehub.resources.inventory import Inventory from ereuse_devicehub.resources.tag.model import Tag from ereuse_devicehub.resources.user import User @@ -32,16 +31,21 @@ class Dummy: @click.confirmation_option(prompt='This command deletes the DB in the process. ' 'Do you want to continue?') def run(self): - print('Preparing the database...') + print('Preparing the database...'.ljust(30), end='') with click_spinner.spinner(): self.app.init_db(erase=True) user = self.user_client('user@dhub.com', '1234') user.post(res=Tag, query=[('ids', i) for i in self.TAGS], data={}) - print('Creating devices...') - for file_name in tqdm(self.SNAPSHOTS): - snapshot = self.file(file_name) - user.post(res=Snapshot, data=snapshot) - print('Done :-)') + files = tuple(Path(__file__).parent.joinpath('files').iterdir()) + print('done.') + with click.progressbar(files, label='Creating devices...'.ljust(28)) as bar: + for path in bar: + with path.open() as f: + snapshot = yaml.load(f) + user.post(res=Snapshot, data=snapshot) + inventory, _ = user.get(res=Inventory) + assert len(inventory['devices']) + print('⭐ Done.') def user_client(self, email: str, password: str): user = User(email=email, password=password) @@ -53,10 +57,3 @@ class Dummy: password=password) client.user, _ = client.login(client.email, client.password) return client - - def file(self, name: str): - with Path(__file__) \ - .parent \ - .joinpath('files') \ - .joinpath(name + '.snapshot.yaml').open() as f: - return yaml.load(f) diff --git a/ereuse_devicehub/dummy/files/real-hp.snapshot.11.yaml b/ereuse_devicehub/dummy/files/real-hp.snapshot.11.yaml new file mode 100644 index 00000000..fa50f2f9 --- /dev/null +++ b/ereuse_devicehub/dummy/files/real-hp.snapshot.11.yaml @@ -0,0 +1,160 @@ +{ + "closed": false, + "uuid": "f9e5e587-baee-44e1-9a94-255d216bbda9", + "components": [ + { + "events": [], + "serialNumber": "6c:62:6d:81:22:9f", + "type": "NetworkAdapter", + "manufacturer": "Intel Corporation", + "speed": 1000, + "model": "82578DM Gigabit Network Connection" + }, + { + "format": "DIMM", + "events": [ + ], + "interface": "DDR3", + "serialNumber": "B4012F30", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "format": "DIMM", + "events": [], + "interface": "DDR3", + "serialNumber": "8E9F2E29", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "format": "DIMM", + "events": [], + "interface": "DDR3", + "serialNumber": "9A012F30", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "format": "DIMM", + "events": [ + ], + "interface": "DDR3", + "serialNumber": "8F9F2E29", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "cores": 2, + "events": [ + { + "rate": 23410.76, + "type": "BenchmarkProcessor", + "elapsed": 1 + }, + { + "rate": 17.0186, + "type": "BenchmarkProcessorSysbench", + "elapsed": 18 + } + ], + "serialNumber": null, + "type": "Processor", + "manufacturer": "Intel Corp.", + "address": 64, + "speed": 1.199, + "model": "Intel Core i3 CPU 530 @ 2.93GHz" + }, + { + "events": [ + { + "writeSpeed": 24.5, + "readSpeed": 111.0, + "type": "BenchmarkDataStorage", + "elapsed": 14 + }, + { + "status": "Unspecified Error. Self-test not started.", + "error": true, + "type": "TestDataStorage", + "elapsed": 1, + "length": Short + } + ], + "interface": "ATA", + "serialNumber": "WD-WCAV2U909540", + "size": 305245, + "type": "HardDrive", + "manufacturer": "Western Digital", + "model": "WDC WD3200AAJS-6" + }, + { + "events": [], + "serialNumber": null, + "type": "SoundCard", + "manufacturer": "Intel Corporation", + "model": "5 Series/3400 Series Chipset High Definition Audio" + }, + { + "memory": 256.0, + "events": [], + "serialNumber": null, + "type": "GraphicCard", + "manufacturer": "Intel Corporation", + "model": "Core Processor Integrated Graphics Controller" + }, + { + "events": [], + "firewire": 0, + "pcmcia": 0, + "serialNumber": "CZC0408YJG", + "slots": 0, + "type": "Motherboard", + "manufacturer": "Hewlett-Packard", + "usb": 2, + "serial": 0, + "model": "304Ah" + } + ], + "software": "Workbench", + "elapsed": 96, + "version": "11.0a1", + "expectedEvents": [ + "StressTest", + "Benchmark", + "SmartTest" + ], + "device": { + "events": [ + { + "type": "StressTest", + "elapsed": 60, + "error": false + }, + { + "rate": 0.9759, + "type": "BenchmarkRamSysbench", + "elapsed": 1 + } + ], + "serialNumber": "CZC0408YJG", + "type": "Desktop", + "manufacturer": "Hewlett-Packard", + "chassis": "Tower", + "model": "HP Compaq 8100 Elite SFF" + }, + "type": "Snapshot", + "date": "2018-06-29T12:28:54.508266" +} diff --git a/ereuse_devicehub/dummy/files/real-toshiba.snapshot.11.yaml b/ereuse_devicehub/dummy/files/real-toshiba.snapshot.11.yaml new file mode 100644 index 00000000..7a6268bd --- /dev/null +++ b/ereuse_devicehub/dummy/files/real-toshiba.snapshot.11.yaml @@ -0,0 +1,139 @@ +{ + "expectedEvents": [ + "SmartTest", + "Benchmark", + "StressTest" + ], + "date": "2018-06-29T15:29:29.322424", + "elapsed": 391, + "software": "Workbench", + "components": [ + { + "serialNumber": null, + "interface": "DDR", + "size": 1024, + "manufacturer": null, + "format": "SODIMM", + "events": [ + ], + "speed": 533.0, + "type": "RamModule", + "model": null + }, + { + "serialNumber": null, + "manufacturer": "Intel Corporation", + "events": [], + "type": "SoundCard", + "model": "NM10/ICH7 Family High Definition Audio Controller" + }, + { + "serialNumber": null, + "manufacturer": "Chicony Electronics Co., Ltd.", + "events": [], + "type": "SoundCard", + "model": "USB2.0 UVC WebCam" + }, + { + "serialNumber": "00:23:08:a5:07:6d", + "manufacturer": "Qualcomm Atheros", + "events": [], + "type": "NetworkAdapter", + "model": "AR9285 Wireless Network Adapter" + }, + { + "serialNumber": "00:23:5a:fe:d7:14", + "manufacturer": "Realtek Semiconductor Co., Ltd.", + "events": [], + "speed": 100, + "type": "NetworkAdapter", + "model": "RTL810xE PCI Express Fast Ethernet controller" + }, + { + "serialNumber": null, + "address": 32, + "manufacturer": "Intel Corp.", + "events": [ + { + "rate": 171.3049, + "type": "BenchmarkProcessorSysbench", + "elapsed": 171 + }, + { + "rate": 6383.9, + "type": "BenchmarkProcessor", + "elapsed": 1 + } + ], + "speed": 1.3330000000000002, + "type": "Processor", + "model": "Intel Atom CPU N270 @ 1.60GHz" + }, + { + "memory": 256.0, + "serialNumber": null, + "manufacturer": "Intel Corporation", + "events": [], + "type": "GraphicCard", + "model": "Mobile 945GSE Express Integrated Graphics Controller" + }, + { + "serialNumber": "090623PB5B00QCGREMAH", + "interface": "ATA", + "size": 152627, + "manufacturer": "Hitachi", + "events": [ + { + "elapsed": 71, + "readSpeed": 19.1, + "type": "BenchmarkDataStorage", + "writeSpeed": 4.7 + }, + { + "elapsed": 2, + "status": "Unspecified Error. Self-test not started.", + "type": "TestDataStorage", + "length": Short, + "error": true + } + ], + "type": "HardDrive", + "model": "HTS54501" + }, + { + "slots": 0, + "serialNumber": "0123456789AB", + "manufacturer": "TOSHIBA", + "firewire": 0, + "events": [], + "pcmcia": 0, + "usb": 5, + "serial": 1, + "type": "Motherboard", + "model": "KAVAA" + } + ], + "version": "11.0a1", + "device": { + "serialNumber": "79545417K", + "manufacturer": "TOSHIBA", + "events": [ + { + "type": "StressTest", + "error": false, + "elapsed": 120 + }, + { + "rate": 19.0443, + "type": "BenchmarkRamSysbench", + "elapsed": 19 + } + ], + "type": "Laptop", + "chassis": "Netbook", + "model": "NB200" + }, + "uuid": "918726ae-c6bc-40aa-97cf-ad80d69268f9", + "closed": false, + "type": "Snapshot" +} diff --git a/ereuse_devicehub/dummy/files/workbench-server-1.snapshot.yaml b/ereuse_devicehub/dummy/files/workbench-server-1.snapshot.yaml index 9f66e2b0..36a4dbd2 100644 --- a/ereuse_devicehub/dummy/files/workbench-server-1.snapshot.yaml +++ b/ereuse_devicehub/dummy/files/workbench-server-1.snapshot.yaml @@ -60,7 +60,6 @@ components: elapsed: 21 - type: TestDataStorage elapsed: 233 - firstError: 0 error: False status: Completed without error length: Short @@ -76,17 +75,14 @@ components: remainingLifetimePercentage: 1 - type: EraseSectors error: False - cleanWithZeros: False + zeros: False startTime: 2018-01-01T10:10:10 endTime: 2018-01-01T12:10:10 - secureRandomSteps: 0 steps: - type: StepRandom startTime: 2018-01-01T10:10:10 endTime: 2018-01-01T12:10:10 error: False - cleanWithZeros: False - secureRandomSteps: 0 - type: HardDrive serialNumber: hdd1-1s model: hdd1-1ml @@ -97,17 +93,14 @@ components: writeSpeed: 5 - type: EraseSectors error: False - cleanWithZeros: False + zeros: False startTime: 2018-01-01T10:10:10 endTime: 2018-01-01T12:10:10 - secureRandomSteps: 0 steps: - type: StepRandom startTime: 2018-01-01T10:10:10 endTime: 2018-01-01T12:10:10 error: False - cleanWithZeros: False - secureRandomSteps: 0 - type: Install elapsed: 420 error: False diff --git a/ereuse_devicehub/resources/device/__init__.py b/ereuse_devicehub/resources/device/__init__.py index 6c879fbb..f319c363 100644 --- a/ereuse_devicehub/resources/device/__init__.py +++ b/ereuse_devicehub/resources/device/__init__.py @@ -1,7 +1,7 @@ from ereuse_devicehub.resources.device.schemas import Cellphone, Component, Computer, \ ComputerMonitor, DataStorage, Desktop, Device, Display, GraphicCard, HardDrive, Laptop, Mobile, \ Monitor, Motherboard, NetworkAdapter, Processor, RamModule, Server, Smartphone, \ - SolidStateDrive, Tablet, TelevisionSet + SolidStateDrive, SoundCard, Tablet, TelevisionSet from ereuse_devicehub.resources.device.views import DeviceView from teal.resource import Converters, Resource @@ -113,6 +113,11 @@ class ProcessorDef(ComponentDef): SCHEMA = Processor +class SoundCardDef(ComponentDef): + VIEW = None + SCHEMA = SoundCard + + class DisplayDef(ComponentDef): VIEW = None SCHEMA = Display diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py index 58bf207f..c50e90af 100644 --- a/ereuse_devicehub/resources/device/models.py +++ b/ereuse_devicehub/resources/device/models.py @@ -1,8 +1,11 @@ from contextlib import suppress -from itertools import chain from operator import attrgetter from typing import Dict, Set +from ereuse_devicehub.resources.enums import ComputerChassis, DataStorageInterface, DisplayTech, \ + RamFormat, RamInterface +from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing +from itertools import chain from sqlalchemy import BigInteger, Column, Enum as DBEnum, Float, ForeignKey, Integer, Sequence, \ SmallInteger, Unicode, inspect from sqlalchemy.ext.declarative import declared_attr @@ -10,14 +13,11 @@ from sqlalchemy.orm import ColumnProperty, backref, relationship, validates from sqlalchemy.util import OrderedSet from sqlalchemy_utils import ColorType from stdnum import imei, meid - -from ereuse_devicehub.resources.enums import ComputerChassis, DataStorageInterface, DisplayTech, \ - RamFormat, RamInterface -from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing -from ereuse_utils.naming import Naming from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, check_range from teal.marshmallow import ValidationError +from ereuse_utils.naming import Naming + class Device(Thing): """ @@ -264,14 +264,14 @@ class SolidStateDrive(DataStorage): class Motherboard(JoinedComponentTableMixin, Component): - slots = Column(SmallInteger, check_range('slots')) + slots = Column(SmallInteger, check_range('slots', min=0)) slots.comment = """ PCI slots the motherboard has. """ - usb = Column(SmallInteger, check_range('usb')) - firewire = Column(SmallInteger, check_range('firewire')) - serial = Column(SmallInteger, check_range('serial')) - pcmcia = Column(SmallInteger, check_range('pcmcia')) + usb = Column(SmallInteger, check_range('usb', min=0)) + firewire = Column(SmallInteger, check_range('firewire', min=0)) + serial = Column(SmallInteger, check_range('serial', min=0)) + pcmcia = Column(SmallInteger, check_range('pcmcia', min=0)) class NetworkMixin: @@ -298,6 +298,10 @@ class RamModule(JoinedComponentTableMixin, Component): format = Column(DBEnum(RamFormat)) +class SoundCard(JoinedComponentTableMixin, Component): + pass + + class Display(JoinedComponentTableMixin, DisplayMixin, Component): """ The display of a device. This is used in all devices that have diff --git a/ereuse_devicehub/resources/device/schemas.py b/ereuse_devicehub/resources/device/schemas.py index c64d6fdd..8380667b 100644 --- a/ereuse_devicehub/resources/device/schemas.py +++ b/ereuse_devicehub/resources/device/schemas.py @@ -1,15 +1,14 @@ +from ereuse_devicehub.marshmallow import NestedOn +from ereuse_devicehub.resources.device import models as m +from ereuse_devicehub.resources.enums import ComputerChassis, DataStorageInterface, DisplayTech, \ + RamFormat, RamInterface +from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE +from ereuse_devicehub.resources.schemas import Thing, UnitCodes from marshmallow import post_load, pre_load from marshmallow.fields import Float, Integer, Str from marshmallow.validate import Length, OneOf, Range from sqlalchemy.util import OrderedSet from stdnum import imei, meid - -from ereuse_devicehub.marshmallow import NestedOn -from ereuse_devicehub.resources.device import models as m -from ereuse_devicehub.resources.enums import ComputerChassis, DisplayTech, \ - RamFormat, RamInterface -from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE -from ereuse_devicehub.resources.schemas import Thing, UnitCodes from teal.marshmallow import EnumField, ValidationError @@ -138,6 +137,7 @@ class DataStorage(Component): size = Integer(validate=Range(0, 10 ** 8), unit=UnitCodes.mbyte, description=m.DataStorage.size.comment) + interface = EnumField(DataStorageInterface) class HardDrive(DataStorage): @@ -149,7 +149,7 @@ class SolidStateDrive(DataStorage): class Motherboard(Component): - slots = Integer(validate=Range(1, 20), + slots = Integer(validate=Range(0, 20), description=m.Motherboard.slots.comment) usb = Integer(validate=Range(0, 20)) firewire = Integer(validate=Range(0, 20)) @@ -174,5 +174,9 @@ class RamModule(Component): format = EnumField(RamFormat) +class SoundCard(Component): + pass + + class Display(DisplayMixin, Component): pass diff --git a/ereuse_devicehub/resources/enums.py b/ereuse_devicehub/resources/enums.py index 9b252871..2154c1c3 100644 --- a/ereuse_devicehub/resources/enums.py +++ b/ereuse_devicehub/resources/enums.py @@ -116,9 +116,11 @@ class ImageMimeTypes(Enum): @unique class SnapshotExpectedEvents(Enum): """Events that Workbench can perform when processing a device.""" + Benchmark = 'Benchmark' TestDataStorage = 'TestDataStorage' StressTest = 'StressTest' EraseSectors = 'EraseSectors' + SmartTest = 'SmartTest' Install = 'Install' diff --git a/ereuse_devicehub/resources/event/models.py b/ereuse_devicehub/resources/event/models.py index 96206b69..f795b518 100644 --- a/ereuse_devicehub/resources/event/models.py +++ b/ereuse_devicehub/resources/event/models.py @@ -1,17 +1,8 @@ from collections import Iterable +from datetime import timedelta from typing import Set, Union from uuid import uuid4 -from flask import g -from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, DateTime, Enum as DBEnum, \ - Float, ForeignKey, Interval, JSON, SmallInteger, Unicode, event -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.ext.orderinglist import ordering_list -from sqlalchemy.orm import backref, relationship -from sqlalchemy.orm.events import AttributeEvents as Events -from sqlalchemy.util import OrderedSet - from ereuse_devicehub.db import db from ereuse_devicehub.resources.device.models import Component, Computer, DataStorage, Device from ereuse_devicehub.resources.enums import AppearanceRange, BOX_RATE_3, BOX_RATE_5, Bios, \ @@ -20,6 +11,15 @@ from ereuse_devicehub.resources.enums import AppearanceRange, BOX_RATE_3, BOX_RA from ereuse_devicehub.resources.image.models import Image from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing from ereuse_devicehub.resources.user.models import User +from flask import g +from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, DateTime, Enum as DBEnum, \ + Float, ForeignKey, Interval, JSON, SmallInteger, Unicode, event +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.ext.orderinglist import ordering_list +from sqlalchemy.orm import backref, relationship, validates +from sqlalchemy.orm.events import AttributeEvents as Events +from sqlalchemy.util import OrderedSet from teal.db import ArrayOfEnum, CASCADE, CASCADE_OWN, INHERIT_COND, POLYMORPHIC_ID, \ POLYMORPHIC_ON, StrictVersionType, check_range @@ -198,10 +198,11 @@ class Deallocate(JoinedTableMixin, EventWithMultipleDevices): class EraseBasic(JoinedTableMixin, EventWithOneDevice): start_time = Column(DateTime, nullable=False) end_time = Column(DateTime, CheckConstraint('end_time > start_time'), nullable=False) - secure_random_steps = Column(SmallInteger, - check_range('secure_random_steps', min=0), - nullable=False) - clean_with_zeros = Column(Boolean, nullable=False) + zeros = Column(Boolean, nullable=False) + zeros.comment = """ + Whether this erasure had a first erasure step consisting of + only writing zeros. + """ class Ready(EventWithMultipleDevices): @@ -219,10 +220,6 @@ class Step(db.Model): error = Column(Boolean, default=False, nullable=False) start_time = Column(DateTime, nullable=False) end_time = Column(DateTime, CheckConstraint('end_time > start_time'), nullable=False) - secure_random_steps = Column(SmallInteger, - check_range('secure_random_steps', min=0), - nullable=False) - clean_with_zeros = Column(Boolean, nullable=False) erasure = relationship(EraseBasic, backref=backref('steps', @@ -412,8 +409,7 @@ class TestDataStorage(Test): id = Column(UUID(as_uuid=True), ForeignKey(Test.id), primary_key=True) length = Column(DBEnum(TestHardDriveLength), nullable=False) # todo from type status = Column(Unicode(STR_SIZE), nullable=False) - lifetime = Column(Interval, nullable=False) - first_error = Column(SmallInteger, nullable=False, default=0) + lifetime = Column(Interval) assessment = Column(Boolean) reallocated_sector_count = Column(SmallInteger) power_cycle_count = Column(SmallInteger) @@ -429,6 +425,10 @@ class TestDataStorage(Test): class StressTest(Test): pass + @validates('elapsed') + def bigger_than_a_minute(self, _, value: timedelta): + assert value.total_seconds() >= 60 + class Benchmark(JoinedTableMixin, EventWithOneDevice): elapsed = Column(Interval) @@ -483,6 +483,13 @@ def validate_device_is_data_storage(target: Event, value: DataStorage, old_value raise TypeError('{} must be a DataStorage but you passed {}'.format(initiator.impl, value)) +@event.listens_for(BenchmarkRamSysbench.device, Events.set.__name__, propagate=True) +def events_not_for_components(target: Event, value: Device, old_value, initiator): + """Validates events that cannot be performed to components.""" + if isinstance(value, Component): + raise TypeError('{!r} cannot be performed to a component ({!r}).'.format(target, value)) + + # The following listeners keep relationships with device <-> components synced with the event # So, if you add or remove devices from events these listeners will # automatically add/remove the ``components`` and ``parent`` of such events diff --git a/ereuse_devicehub/resources/event/models.pyi b/ereuse_devicehub/resources/event/models.pyi index 48d86401..e24a1e42 100644 --- a/ereuse_devicehub/resources/event/models.pyi +++ b/ereuse_devicehub/resources/event/models.pyi @@ -81,8 +81,6 @@ class Step(Model): self.success = ... # type: bool self.start_time = ... # type: datetime self.end_time = ... # type: datetime - self.secure_random_steps = ... # type: int - self.clean_with_zeros = ... # type: bool self.erasure = ... # type: EraseBasic @@ -230,9 +228,8 @@ class EraseBasic(EventWithOneDevice): super().__init__(**kwargs) self.start_time = ... # type: datetime self.end_time = ... # type: datetime - self.secure_random_steps = ... # type: int self.steps = ... # type: List[Step] - self.clean_with_zeros = ... # type: bool + self.zeros = ... # type: bool self.success = ... # type: bool diff --git a/ereuse_devicehub/resources/event/schemas.py b/ereuse_devicehub/resources/event/schemas.py index fad0bc61..88de9875 100644 --- a/ereuse_devicehub/resources/event/schemas.py +++ b/ereuse_devicehub/resources/event/schemas.py @@ -63,9 +63,7 @@ class Deallocate(EventWithMultipleDevices): class EraseBasic(EventWithOneDevice): start_time = DateTime(required=True, data_key='startTime') end_time = DateTime(required=True, data_key='endTime') - secure_random_steps = Integer(validate=Range(min=0), required=True, - data_key='secureRandomSteps') - clean_with_zeros = Boolean(required=True, data_key='cleanWithZeros') + zeros = Boolean(required=True, description=m.EraseBasic.zeros.comment) steps = NestedOn('Step', many=True, required=True) @@ -77,10 +75,6 @@ class Step(Schema): type = String(description='Only required when it is nested.') start_time = DateTime(required=True, data_key='startTime') end_time = DateTime(required=True, data_key='endTime') - secure_random_steps = Integer(validate=Range(min=0), - required=True, - data_key='secureRandomSteps') - clean_with_zeros = Boolean(required=True, data_key='cleanWithZeros') error = Boolean(default=False, description='Did the event fail?') @@ -243,8 +237,7 @@ class Test(EventWithOneDevice): class TestDataStorage(Test): length = EnumField(TestHardDriveLength, required=True) status = String(validate=Length(max=STR_SIZE), required=True) - lifetime = TimeDelta(precision=TimeDelta.DAYS, required=True) - first_error = Integer(missing=0, data_key='firstError') + lifetime = TimeDelta(precision=TimeDelta.DAYS) assessment = Boolean() reallocated_sector_count = Integer(data_key='reallocatedSectorCount') power_cycle_count = Integer(data_key='powerCycleCount') diff --git a/setup.py b/setup.py index 747b8d3d..0bb1fe74 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open('README.md') as f: setup( name='ereuse-devicehub', - version='0.2.0a9', + version='0.2.0a10', packages=find_packages(), url='https://github.com/ereuse/devicehub-teal', license='Affero', @@ -16,14 +16,14 @@ setup( long_description=long_description, long_description_content_type='text/markdown', install_requires=[ - 'teal>=0.2.0a5', + 'teal>=0.2.0a6', 'marshmallow_enum', 'ereuse-utils[Naming]>=0.3.0b2', 'psycopg2-binary', 'requests', 'requests-toolbelt', 'hashids', - 'tqdm', + 'click', 'click-spinner', 'sqlalchemy-utils[password, color, babel]', 'PyYAML', diff --git a/tests/files/erase-sectors.snapshot.yaml b/tests/files/erase-sectors.snapshot.yaml index b4275b23..f08f1575 100644 --- a/tests/files/erase-sectors.snapshot.yaml +++ b/tests/files/erase-sectors.snapshot.yaml @@ -16,23 +16,18 @@ components: manufacturer: c1mr events: - type: EraseSectors - cleanWithZeros: True + zeros: True startTime: 2018-06-01T08:12:06 endTime: 2018-06-01T09:12:06 - secureRandomSteps: 20 steps: - type: StepZero error: False startTime: 2018-06-01T08:15:00 endTime: 2018-06-01T09:16:00 - secureRandomSteps: 1 - cleanWithZeros: True - type: StepZero error: False startTime: 2018-06-01T08:16:00 endTime: 2018-06-01T09:17:00 - secureRandomSteps: 1 - cleanWithZeros: True - type: GraphicCard serialNumber: gc1s model: gc1ml diff --git a/tests/files/real-hp.snapshot.11.yaml b/tests/files/real-hp.snapshot.11.yaml new file mode 100644 index 00000000..fa50f2f9 --- /dev/null +++ b/tests/files/real-hp.snapshot.11.yaml @@ -0,0 +1,160 @@ +{ + "closed": false, + "uuid": "f9e5e587-baee-44e1-9a94-255d216bbda9", + "components": [ + { + "events": [], + "serialNumber": "6c:62:6d:81:22:9f", + "type": "NetworkAdapter", + "manufacturer": "Intel Corporation", + "speed": 1000, + "model": "82578DM Gigabit Network Connection" + }, + { + "format": "DIMM", + "events": [ + ], + "interface": "DDR3", + "serialNumber": "B4012F30", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "format": "DIMM", + "events": [], + "interface": "DDR3", + "serialNumber": "8E9F2E29", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "format": "DIMM", + "events": [], + "interface": "DDR3", + "serialNumber": "9A012F30", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "format": "DIMM", + "events": [ + ], + "interface": "DDR3", + "serialNumber": "8F9F2E29", + "size": 2048, + "type": "RamModule", + "manufacturer": "JEDEC ID:80 2C", + "speed": 1333.0, + "model": "16JTF25664AZ-1G4F" + }, + { + "cores": 2, + "events": [ + { + "rate": 23410.76, + "type": "BenchmarkProcessor", + "elapsed": 1 + }, + { + "rate": 17.0186, + "type": "BenchmarkProcessorSysbench", + "elapsed": 18 + } + ], + "serialNumber": null, + "type": "Processor", + "manufacturer": "Intel Corp.", + "address": 64, + "speed": 1.199, + "model": "Intel Core i3 CPU 530 @ 2.93GHz" + }, + { + "events": [ + { + "writeSpeed": 24.5, + "readSpeed": 111.0, + "type": "BenchmarkDataStorage", + "elapsed": 14 + }, + { + "status": "Unspecified Error. Self-test not started.", + "error": true, + "type": "TestDataStorage", + "elapsed": 1, + "length": Short + } + ], + "interface": "ATA", + "serialNumber": "WD-WCAV2U909540", + "size": 305245, + "type": "HardDrive", + "manufacturer": "Western Digital", + "model": "WDC WD3200AAJS-6" + }, + { + "events": [], + "serialNumber": null, + "type": "SoundCard", + "manufacturer": "Intel Corporation", + "model": "5 Series/3400 Series Chipset High Definition Audio" + }, + { + "memory": 256.0, + "events": [], + "serialNumber": null, + "type": "GraphicCard", + "manufacturer": "Intel Corporation", + "model": "Core Processor Integrated Graphics Controller" + }, + { + "events": [], + "firewire": 0, + "pcmcia": 0, + "serialNumber": "CZC0408YJG", + "slots": 0, + "type": "Motherboard", + "manufacturer": "Hewlett-Packard", + "usb": 2, + "serial": 0, + "model": "304Ah" + } + ], + "software": "Workbench", + "elapsed": 96, + "version": "11.0a1", + "expectedEvents": [ + "StressTest", + "Benchmark", + "SmartTest" + ], + "device": { + "events": [ + { + "type": "StressTest", + "elapsed": 60, + "error": false + }, + { + "rate": 0.9759, + "type": "BenchmarkRamSysbench", + "elapsed": 1 + } + ], + "serialNumber": "CZC0408YJG", + "type": "Desktop", + "manufacturer": "Hewlett-Packard", + "chassis": "Tower", + "model": "HP Compaq 8100 Elite SFF" + }, + "type": "Snapshot", + "date": "2018-06-29T12:28:54.508266" +} diff --git a/tests/files/real-toshiba.snapshot.11.yaml b/tests/files/real-toshiba.snapshot.11.yaml new file mode 100644 index 00000000..7a6268bd --- /dev/null +++ b/tests/files/real-toshiba.snapshot.11.yaml @@ -0,0 +1,139 @@ +{ + "expectedEvents": [ + "SmartTest", + "Benchmark", + "StressTest" + ], + "date": "2018-06-29T15:29:29.322424", + "elapsed": 391, + "software": "Workbench", + "components": [ + { + "serialNumber": null, + "interface": "DDR", + "size": 1024, + "manufacturer": null, + "format": "SODIMM", + "events": [ + ], + "speed": 533.0, + "type": "RamModule", + "model": null + }, + { + "serialNumber": null, + "manufacturer": "Intel Corporation", + "events": [], + "type": "SoundCard", + "model": "NM10/ICH7 Family High Definition Audio Controller" + }, + { + "serialNumber": null, + "manufacturer": "Chicony Electronics Co., Ltd.", + "events": [], + "type": "SoundCard", + "model": "USB2.0 UVC WebCam" + }, + { + "serialNumber": "00:23:08:a5:07:6d", + "manufacturer": "Qualcomm Atheros", + "events": [], + "type": "NetworkAdapter", + "model": "AR9285 Wireless Network Adapter" + }, + { + "serialNumber": "00:23:5a:fe:d7:14", + "manufacturer": "Realtek Semiconductor Co., Ltd.", + "events": [], + "speed": 100, + "type": "NetworkAdapter", + "model": "RTL810xE PCI Express Fast Ethernet controller" + }, + { + "serialNumber": null, + "address": 32, + "manufacturer": "Intel Corp.", + "events": [ + { + "rate": 171.3049, + "type": "BenchmarkProcessorSysbench", + "elapsed": 171 + }, + { + "rate": 6383.9, + "type": "BenchmarkProcessor", + "elapsed": 1 + } + ], + "speed": 1.3330000000000002, + "type": "Processor", + "model": "Intel Atom CPU N270 @ 1.60GHz" + }, + { + "memory": 256.0, + "serialNumber": null, + "manufacturer": "Intel Corporation", + "events": [], + "type": "GraphicCard", + "model": "Mobile 945GSE Express Integrated Graphics Controller" + }, + { + "serialNumber": "090623PB5B00QCGREMAH", + "interface": "ATA", + "size": 152627, + "manufacturer": "Hitachi", + "events": [ + { + "elapsed": 71, + "readSpeed": 19.1, + "type": "BenchmarkDataStorage", + "writeSpeed": 4.7 + }, + { + "elapsed": 2, + "status": "Unspecified Error. Self-test not started.", + "type": "TestDataStorage", + "length": Short, + "error": true + } + ], + "type": "HardDrive", + "model": "HTS54501" + }, + { + "slots": 0, + "serialNumber": "0123456789AB", + "manufacturer": "TOSHIBA", + "firewire": 0, + "events": [], + "pcmcia": 0, + "usb": 5, + "serial": 1, + "type": "Motherboard", + "model": "KAVAA" + } + ], + "version": "11.0a1", + "device": { + "serialNumber": "79545417K", + "manufacturer": "TOSHIBA", + "events": [ + { + "type": "StressTest", + "error": false, + "elapsed": 120 + }, + { + "rate": 19.0443, + "type": "BenchmarkRamSysbench", + "elapsed": 19 + } + ], + "type": "Laptop", + "chassis": "Netbook", + "model": "NB200" + }, + "uuid": "918726ae-c6bc-40aa-97cf-ad80d69268f9", + "closed": false, + "type": "Snapshot" +} diff --git a/tests/files/workbench-server-1.snapshot.yaml b/tests/files/workbench-server-1.snapshot.yaml index e1b7c71a..adbedb9b 100644 --- a/tests/files/workbench-server-1.snapshot.yaml +++ b/tests/files/workbench-server-1.snapshot.yaml @@ -58,7 +58,6 @@ components: elapsed: 21 - type: TestDataStorage elapsed: 233 - firstError: 0 error: False status: Completed without error length: Short diff --git a/tests/files/workbench-server-3.erase.yaml b/tests/files/workbench-server-3.erase.yaml index 4ef2b792..c8862d88 100644 --- a/tests/files/workbench-server-3.erase.yaml +++ b/tests/files/workbench-server-3.erase.yaml @@ -10,14 +10,11 @@ type: 'EraseSectors' error: False # snapshot: None fulfill! # device: None fulfill! -cleanWithZeros: False +zeros: False startTime: 2018-01-01T10:10:10 endTime: 2018-01-01T12:10:10 -secureRandomSteps: 0 steps: - type: 'StepRandom' startTime: '2018-01-01T10:10:10' endTime: '2018-01-01T12:10:10' error: False - cleanWithZeros: False - secureRandomSteps: 0 diff --git a/tests/test_basic.py b/tests/test_basic.py index 9873cb0d..f864ce15 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -35,4 +35,4 @@ def test_api_docs(client: Client): 'scheme': 'basic', 'name': 'Authorization' } - assert len(docs['definitions']) == 51 + assert 52 == len(docs['definitions']) diff --git a/tests/test_event.py b/tests/test_event.py index b0d8cc64..c926f85a 100644 --- a/tests/test_event.py +++ b/tests/test_event.py @@ -1,15 +1,14 @@ from datetime import datetime, timedelta import pytest -from flask import g -from sqlalchemy.util import OrderedSet - from ereuse_devicehub.db import db from ereuse_devicehub.resources.device.models import Computer, Device, GraphicCard, HardDrive, \ RamModule, SolidStateDrive from ereuse_devicehub.resources.enums import TestHardDriveLength from ereuse_devicehub.resources.event.models import BenchmarkDataStorage, EraseBasic, EraseSectors, \ - EventWithOneDevice, Install, Ready, StepZero, StressTest, TestDataStorage + EventWithOneDevice, Install, Ready, StepRandom, StepZero, StressTest, TestDataStorage +from flask import g +from sqlalchemy.util import OrderedSet from tests.conftest import create_user @@ -34,10 +33,9 @@ def test_author(): def test_erase_basic(): erasure = EraseBasic( device=HardDrive(serial_number='foo', manufacturer='bar', model='foo-bar'), - clean_with_zeros=True, + zeros=True, start_time=datetime.now(), end_time=datetime.now(), - secure_random_steps=25, error=False ) db.session.add(erasure) @@ -59,7 +57,6 @@ def test_validate_device_data_storage(): clean_with_zeros=True, start_time=datetime.now(), end_time=datetime.now(), - secure_random_steps=25, error=False ) @@ -68,38 +65,28 @@ def test_validate_device_data_storage(): def test_erase_sectors_steps(): erasure = EraseSectors( device=SolidStateDrive(serial_number='foo', manufacturer='bar', model='foo-bar'), - clean_with_zeros=True, + zeros=True, start_time=datetime.now(), end_time=datetime.now(), - secure_random_steps=25, error=False, steps=[ StepZero(error=False, start_time=datetime.now(), - end_time=datetime.now(), - secure_random_steps=1, - clean_with_zeros=True), + end_time=datetime.now()), + StepRandom(error=False, + start_time=datetime.now(), + end_time=datetime.now()), StepZero(error=False, start_time=datetime.now(), - end_time=datetime.now(), - secure_random_steps=2, - clean_with_zeros=True), - StepZero(error=False, - start_time=datetime.now(), - end_time=datetime.now(), - secure_random_steps=3, - clean_with_zeros=True) + end_time=datetime.now()) ] ) db.session.add(erasure) db.session.commit() db_erasure = EraseSectors.query.one() # Steps are in order - assert db_erasure.steps[0].secure_random_steps == 1 assert db_erasure.steps[0].num == 0 - assert db_erasure.steps[1].secure_random_steps == 2 assert db_erasure.steps[1].num == 1 - assert db_erasure.steps[2].secure_random_steps == 3 assert db_erasure.steps[2].num == 2 diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 66162008..f3d4d392 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -330,8 +330,6 @@ def test_erase(user: UserClient): for step in erasure['steps']: assert step['type'] == 'StepZero' assert step['error'] is False - assert step['secureRandomSteps'] == 1 - assert step['cleanWithZeros'] is True assert 'num' not in step diff --git a/tests/test_workbench.py b/tests/test_workbench.py index 410924eb..c1382231 100644 --- a/tests/test_workbench.py +++ b/tests/test_workbench.py @@ -112,3 +112,24 @@ def test_workbench_server_phases(user: UserClient): pc, _ = user.get(res=Device, item=snapshot['id']) assert len(pc['events']) == 10 # todo shall I add child events? + + +def test_real_hp_11(user: UserClient): + s = file('real-hp.snapshot.11') + snapshot, _ = user.post(res=Snapshot, data=s) + assert snapshot['device']['hid'] == 'hewlett-packard-czc0408yjg-hp_compaq_8100_elite_sff' + assert snapshot['device']['chassis'] == 'Tower' + assert set(e['type'] for e in snapshot['events']) == { + 'BenchmarkDataStorage', + 'BenchmarkProcessor', + 'BenchmarkProcessorSysbench', + 'TestDataStorage', + 'BenchmarkRamSysbench', + 'StressTest' + } + assert len(list(e['type'] for e in snapshot['events'])) == 6 + + +def test_real_toshiba_11(user: UserClient): + s = file('real-toshiba.snapshot.11') + snapshot, _ = user.post(res=Snapshot, data=s)