From 042b7718ec31e05db71acd36bae1133f45b588db Mon Sep 17 00:00:00 2001 From: Xavier Bustamante Talavera Date: Sun, 30 Sep 2018 12:29:33 +0200 Subject: [PATCH] Use SanitizedStr and CITText, lowering many strings --- ereuse_devicehub/resources/agent/models.py | 11 +++--- ereuse_devicehub/resources/agent/schemas.py | 11 +++--- ereuse_devicehub/resources/device/models.py | 13 +++---- ereuse_devicehub/resources/device/schemas.py | 10 +++--- ereuse_devicehub/resources/event/models.py | 23 +++++++------ ereuse_devicehub/resources/event/schemas.py | 36 +++++++++++--------- ereuse_devicehub/resources/image/models.py | 7 ++-- ereuse_devicehub/resources/lot/models.py | 5 +-- ereuse_devicehub/resources/lot/schemas.py | 3 +- ereuse_devicehub/resources/tag/__init__.py | 3 +- ereuse_devicehub/resources/tag/model.py | 6 ++-- ereuse_devicehub/resources/tag/schema.py | 12 +++---- ereuse_devicehub/resources/user/schemas.py | 5 +-- examples/create-db.sh | 1 + requirements.txt | 3 +- setup.py | 3 +- tests/conftest.py | 2 +- tests/test_agent.py | 12 +++---- tests/test_device.py | 12 +++---- tests/test_event.py | 10 +++--- tests/test_snapshot.py | 2 +- tests/test_tag.py | 9 +++-- tests/test_workbench.py | 16 ++++----- 23 files changed, 118 insertions(+), 97 deletions(-) diff --git a/ereuse_devicehub/resources/agent/models.py b/ereuse_devicehub/resources/agent/models.py index aa53d6e5..6b647dc2 100644 --- a/ereuse_devicehub/resources/agent/models.py +++ b/ereuse_devicehub/resources/agent/models.py @@ -2,6 +2,7 @@ from itertools import chain from operator import attrgetter from uuid import uuid4 +from citext import CIText from flask import current_app as app, g from sqlalchemy import Column, Enum as DBEnum, ForeignKey, Unicode, UniqueConstraint from sqlalchemy.dialects.postgresql import UUID @@ -9,11 +10,11 @@ from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import backref, relationship, validates from sqlalchemy_utils import EmailType, PhoneNumberType from teal import enums -from teal.db import DBError, INHERIT_COND, POLYMORPHIC_ID, POLYMORPHIC_ON +from teal.db import DBError, INHERIT_COND, POLYMORPHIC_ID, POLYMORPHIC_ON, check_lower from teal.marshmallow import ValidationError from werkzeug.exceptions import NotImplemented, UnprocessableEntity -from ereuse_devicehub.resources.models import STR_SIZE, STR_SM_SIZE, Thing +from ereuse_devicehub.resources.models import STR_SM_SIZE, Thing from ereuse_devicehub.resources.user.models import User @@ -27,11 +28,11 @@ class JoinedTableMixin: class Agent(Thing): id = Column(UUID(as_uuid=True), primary_key=True, default=uuid4) type = Column(Unicode, nullable=False) - name = Column(Unicode(length=STR_SM_SIZE)) + name = Column(CIText()) name.comment = """ The name of the organization or person. """ - tax_id = Column(Unicode(length=STR_SM_SIZE)) + tax_id = Column(Unicode(length=STR_SM_SIZE), check_lower('tax_id')) tax_id.comment = """ The Tax / Fiscal ID of the organization, e.g. the TIN in the US or the CIF/NIF in Spain. @@ -111,7 +112,7 @@ class Membership(Thing): For example, because the individual works in or because is a member of. """ - id = Column(Unicode(length=STR_SIZE)) + id = Column(Unicode(), check_lower('id')) organization_id = Column(UUID(as_uuid=True), ForeignKey(Organization.id), primary_key=True) organization = relationship(Organization, backref=backref('members', collection_class=set, lazy=True), diff --git a/ereuse_devicehub/resources/agent/schemas.py b/ereuse_devicehub/resources/agent/schemas.py index 6b5d1365..459319db 100644 --- a/ereuse_devicehub/resources/agent/schemas.py +++ b/ereuse_devicehub/resources/agent/schemas.py @@ -1,7 +1,7 @@ from marshmallow import fields as ma_fields, validate as ma_validate from marshmallow.fields import Email from teal import enums -from teal.marshmallow import EnumField, Phone +from teal.marshmallow import EnumField, Phone, SanitizedStr from ereuse_devicehub.marshmallow import NestedOn from ereuse_devicehub.resources.models import STR_SIZE, STR_SM_SIZE @@ -10,9 +10,10 @@ from ereuse_devicehub.resources.schemas import Thing class Agent(Thing): id = ma_fields.UUID(dump_only=True) - name = ma_fields.String(validate=ma_validate.Length(max=STR_SM_SIZE)) - tax_id = ma_fields.String(validate=ma_validate.Length(max=STR_SM_SIZE), - data_key='taxId') + name = SanitizedStr(validate=ma_validate.Length(max=STR_SM_SIZE)) + tax_id = SanitizedStr(lower=True, + validate=ma_validate.Length(max=STR_SM_SIZE), + data_key='taxId') country = EnumField(enums.Country) telephone = Phone() email = Email() @@ -25,7 +26,7 @@ class Organization(Agent): class Membership(Thing): organization = NestedOn(Organization) individual = NestedOn('Individual') - id = ma_fields.String(validate=ma_validate.Length(max=STR_SIZE)) + id = SanitizedStr(lower=True, validate=ma_validate.Length(max=STR_SIZE)) class Individual(Agent): diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py index 644ddf1b..ffb8eecd 100644 --- a/ereuse_devicehub/resources/device/models.py +++ b/ereuse_devicehub/resources/device/models.py @@ -11,12 +11,13 @@ from sqlalchemy.orm import ColumnProperty, backref, relationship, validates from sqlalchemy.util import OrderedSet from sqlalchemy_utils import ColorType from stdnum import imei, meid -from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, check_range +from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, check_lower, \ + check_range from teal.marshmallow import ValidationError from ereuse_devicehub.resources.enums import ComputerChassis, DataStorageInterface, DisplayTech, \ RamFormat, RamInterface -from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing +from ereuse_devicehub.resources.models import STR_SM_SIZE, Thing class Device(Thing): @@ -29,14 +30,14 @@ class Device(Thing): The identifier of the device for this database. """ type = Column(Unicode(STR_SM_SIZE), nullable=False) - hid = Column(Unicode(STR_BIG_SIZE), unique=True) + hid = Column(Unicode(), check_lower('hid'), unique=True) hid.comment = """ The Hardware ID (HID) is the unique ID traceability systems use to ID a device globally. """ - model = Column(Unicode(STR_BIG_SIZE)) - manufacturer = Column(Unicode(STR_SIZE)) - serial_number = Column(Unicode(STR_SIZE)) + model = Column(Unicode(), check_lower('model')) + manufacturer = Column(Unicode(), check_lower('manufacturer')) + serial_number = Column(Unicode(), check_lower('serial_number')) weight = Column(Float(decimal_return_scale=3), check_range('weight', 0.1, 3)) weight.comment = """ The weight of the device in Kgm. diff --git a/ereuse_devicehub/resources/device/schemas.py b/ereuse_devicehub/resources/device/schemas.py index f7eab413..e7160511 100644 --- a/ereuse_devicehub/resources/device/schemas.py +++ b/ereuse_devicehub/resources/device/schemas.py @@ -3,7 +3,7 @@ from marshmallow.fields import Boolean, Float, Integer, Str from marshmallow.validate import Length, OneOf, Range from sqlalchemy.util import OrderedSet from stdnum import imei, meid -from teal.marshmallow import EnumField, ValidationError +from teal.marshmallow import EnumField, SanitizedStr, ValidationError from ereuse_devicehub.marshmallow import NestedOn from ereuse_devicehub.resources.device import models as m @@ -15,14 +15,14 @@ from ereuse_devicehub.resources.schemas import Thing, UnitCodes class Device(Thing): id = Integer(description=m.Device.id.comment, dump_only=True) - hid = Str(dump_only=True, description=m.Device.hid.comment) + hid = SanitizedStr(lower=True, dump_only=True, description=m.Device.hid.comment) tags = NestedOn('Tag', many=True, collection_class=OrderedSet, description='The set of tags that identify the device.') - model = Str(validate=Length(max=STR_BIG_SIZE)) - manufacturer = Str(validate=Length(max=STR_SIZE)) - serial_number = Str(data_key='serialNumber') + model = SanitizedStr(lower=True, validate=Length(max=STR_BIG_SIZE)) + manufacturer = SanitizedStr(lower=True, validate=Length(max=STR_SIZE)) + serial_number = SanitizedStr(lower=True, data_key='serialNumber') weight = Float(validate=Range(0.1, 3), unit=UnitCodes.kgm, description=m.Device.weight.comment) width = Float(validate=Range(0.1, 3), unit=UnitCodes.m, description=m.Device.width.comment) height = Float(validate=Range(0.1, 3), unit=UnitCodes.m, description=m.Device.height.comment) diff --git a/ereuse_devicehub/resources/event/models.py b/ereuse_devicehub/resources/event/models.py index 5c041625..198ea7b0 100644 --- a/ereuse_devicehub/resources/event/models.py +++ b/ereuse_devicehub/resources/event/models.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from typing import Set, Union from uuid import uuid4 +from citext import CIText from flask import current_app as app, g from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, DateTime, Enum as DBEnum, \ Float, ForeignKey, Interval, JSON, Numeric, SmallInteger, Unicode, event, orm @@ -13,7 +14,7 @@ from sqlalchemy.orm import backref, relationship, validates from sqlalchemy.orm.events import AttributeEvents as Events from sqlalchemy.util import OrderedSet from teal.db import ArrayOfEnum, CASCADE, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \ - POLYMORPHIC_ON, StrictVersionType, URL, check_range + POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range from teal.enums import Country, Currency, Subdivision from teal.marshmallow import ValidationError @@ -25,7 +26,7 @@ from ereuse_devicehub.resources.enums import AppearanceRange, BOX_RATE_3, BOX_RA FunctionalityRange, PriceSoftware, RATE_NEGATIVE, RATE_POSITIVE, RatingRange, RatingSoftware, \ ReceiverRole, SnapshotExpectedEvents, SnapshotSoftware, TestHardDriveLength from ereuse_devicehub.resources.image.models import Image -from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing +from ereuse_devicehub.resources.models import STR_SM_SIZE, Thing from ereuse_devicehub.resources.user.models import User """ @@ -43,7 +44,7 @@ class JoinedTableMixin: class Event(Thing): id = Column(UUID(as_uuid=True), primary_key=True, default=uuid4) type = Column(Unicode, nullable=False) - name = Column(Unicode(STR_BIG_SIZE), default='', nullable=False) + name = Column(CIText(), default='', nullable=False) name.comment = """ A name or title for the event. Used when searching for events. """ @@ -263,13 +264,13 @@ class Remove(EventWithOneDevice): class Allocate(JoinedTableMixin, EventWithMultipleDevices): to_id = Column(UUID, ForeignKey(User.id)) to = relationship(User, primaryjoin=User.id == to_id) - organization = Column(Unicode(STR_SIZE)) + organization = Column(CIText()) class Deallocate(JoinedTableMixin, EventWithMultipleDevices): from_id = Column(UUID, ForeignKey(User.id)) from_rel = relationship(User, primaryjoin=User.id == from_id) - organization = Column(Unicode(STR_SIZE)) + organization = Column(CIText()) class EraseBasic(JoinedWithOneDeviceMixin, EventWithOneDevice): @@ -588,7 +589,7 @@ class Test(JoinedWithOneDeviceMixin, EventWithOneDevice): class TestDataStorage(Test): id = Column(UUID(as_uuid=True), ForeignKey(Test.id), primary_key=True) length = Column(DBEnum(TestHardDriveLength), nullable=False) # todo from type - status = Column(Unicode(STR_SIZE), nullable=False) + status = Column(Unicode(), check_lower('status'), nullable=False) lifetime = Column(Interval) assessment = Column(Boolean) reallocated_sector_count = Column(SmallInteger) @@ -681,13 +682,13 @@ class Live(JoinedWithOneDeviceMixin, EventWithOneDevice): check_range('subdivision_confidence', 0, 100), nullable=False) subdivision = Column(DBEnum(Subdivision), nullable=False) - city = Column(Unicode(STR_SM_SIZE), nullable=False) + city = Column(Unicode(STR_SM_SIZE), check_lower('city'), nullable=False) city_confidence = Column(SmallInteger, check_range('city_confidence', 0, 100), nullable=False) - isp = Column(Unicode(length=STR_SM_SIZE), nullable=False) - organization = Column(Unicode(length=STR_SIZE)) - organization_type = Column(Unicode(length=STR_SM_SIZE)) + isp = Column(Unicode(STR_SM_SIZE), check_lower('isp'), nullable=False) + organization = Column(Unicode(STR_SM_SIZE), check_lower('organization')) + organization_type = Column(Unicode(STR_SM_SIZE), check_lower('organization_type')) @property def country(self) -> Country: @@ -713,7 +714,7 @@ class Trade(JoinedTableMixin, EventWithMultipleDevices): shipping_date.comment = """ When are the devices going to be ready for shipping? """ - invoice_number = Column(Unicode(length=STR_SIZE)) + invoice_number = Column(CIText()) invoice_number.comment = """ The id of the invoice so they can be linked. """ diff --git a/ereuse_devicehub/resources/event/schemas.py b/ereuse_devicehub/resources/event/schemas.py index 372fd7d4..ad246617 100644 --- a/ereuse_devicehub/resources/event/schemas.py +++ b/ereuse_devicehub/resources/event/schemas.py @@ -7,7 +7,7 @@ from marshmallow.fields import Boolean, DateTime, Decimal, Float, Integer, List, from marshmallow.validate import Length, Range from sqlalchemy.util import OrderedSet from teal.enums import Country, Currency, Subdivision -from teal.marshmallow import EnumField, IP, Version +from teal.marshmallow import EnumField, IP, SanitizedStr, Version from teal.resource import Schema from ereuse_devicehub.marshmallow import NestedOn @@ -24,11 +24,13 @@ from ereuse_devicehub.resources.user.schemas import User class Event(Thing): id = UUID(dump_only=True) - name = String(default='', validate=Length(max=STR_BIG_SIZE), description=m.Event.name.comment) + name = SanitizedStr(default='', + validate=Length(max=STR_BIG_SIZE), + description=m.Event.name.comment) incidence = Boolean(default=False, description=m.Event.incidence.comment) closed = Boolean(missing=True, description=m.Event.closed.comment) error = Boolean(default=False, description=m.Event.error.comment) - description = String(default='', description=m.Event.description.comment) + description = SanitizedStr(default='', description=m.Event.description.comment) start_time = DateTime(data_key='startTime', description=m.Event.start_time.comment) end_time = DateTime(data_key='endTime', description=m.Event.end_time.comment) snapshot = NestedOn('Snapshot', dump_only=True) @@ -57,16 +59,18 @@ class Remove(EventWithOneDevice): class Allocate(EventWithMultipleDevices): to = NestedOn(User, description='The user the devices are allocated to.') - organization = String(validate=Length(max=STR_SIZE), - description='The organization where the user was when this happened.') + organization = SanitizedStr(validate=Length(max=STR_SIZE), + description='The organization where the ' + 'user was when this happened.') class Deallocate(EventWithMultipleDevices): from_rel = Nested(User, data_key='from', description='The user where the devices are not allocated to anymore.') - organization = String(validate=Length(max=STR_SIZE), - description='The organization where the user was when this happened.') + organization = SanitizedStr(validate=Length(max=STR_SIZE), + description='The organization where the ' + 'user was when this happened.') class EraseBasic(EventWithOneDevice): @@ -187,9 +191,9 @@ class EreusePrice(Price): class Install(EventWithOneDevice): - name = String(validate=Length(min=4, max=STR_BIG_SIZE), - required=True, - description='The name of the OS installed.') + name = SanitizedStr(validate=Length(min=4, max=STR_BIG_SIZE), + required=True, + description='The name of the OS installed.') elapsed = TimeDelta(precision=TimeDelta.SECONDS, required=True) @@ -263,7 +267,7 @@ class Test(EventWithOneDevice): class TestDataStorage(Test): length = EnumField(TestHardDriveLength, required=True) - status = String(validate=Length(max=STR_SIZE), required=True) + status = SanitizedStr(lower=True, validate=Length(max=STR_SIZE), required=True) lifetime = TimeDelta(precision=TimeDelta.DAYS) assessment = Boolean() reallocated_sector_count = Integer(data_key='reallocatedSectorCount') @@ -329,11 +333,11 @@ class Live(EventWithOneDevice): subdivision_confidence = Integer(dump_only=True, data_key='subdivisionConfidence') subdivision = EnumField(Subdivision, dump_only=True) country = EnumField(Country, dump_only=True) - city = String(dump_only=True) + city = SanitizedStr(lower=True, dump_only=True) city_confidence = Integer(dump_only=True, data_key='cityConfidence') - isp = String(dump_only=True) - organization = String(dump_only=True) - organization_type = String(dump_only=True, data_key='organizationType') + isp = SanitizedStr(lower=True, dump_only=True) + organization = SanitizedStr(lower=True, dump_only=True) + organization_type = SanitizedStr(lower=True, dump_only=True, data_key='organizationType') class Organize(EventWithMultipleDevices): @@ -350,7 +354,7 @@ class CancelReservation(Organize): class Trade(EventWithMultipleDevices): shipping_date = DateTime(data_key='shippingDate') - invoice_number = String(validate=Length(max=STR_SIZE), data_key='invoiceNumber') + invoice_number = SanitizedStr(validate=Length(max=STR_SIZE), data_key='invoiceNumber') price = NestedOn(Price) to = NestedOn(Agent, only_query='id', required=True, comment=m.Trade.to_comment) confirms = NestedOn(Organize) diff --git a/ereuse_devicehub/resources/image/models.py b/ereuse_devicehub/resources/image/models.py index 08f5591a..06f45bad 100644 --- a/ereuse_devicehub/resources/image/models.py +++ b/ereuse_devicehub/resources/image/models.py @@ -1,6 +1,7 @@ from uuid import uuid4 -from sqlalchemy import BigInteger, Column, Enum as DBEnum, ForeignKey, Unicode +from citext import CIText +from sqlalchemy import BigInteger, Column, Enum as DBEnum, ForeignKey from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import backref, relationship from sqlalchemy.util import OrderedSet @@ -9,7 +10,7 @@ from teal.db import CASCADE_OWN from ereuse_devicehub.db import db from ereuse_devicehub.resources.device.models import Device from ereuse_devicehub.resources.enums import ImageMimeTypes, Orientation -from ereuse_devicehub.resources.models import STR_BIG_SIZE, Thing +from ereuse_devicehub.resources.models import Thing class ImageList(Thing): @@ -26,7 +27,7 @@ class ImageList(Thing): class Image(Thing): id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid4) - name = Column(Unicode(STR_BIG_SIZE), default='', nullable=False) + name = Column(CIText(), default='', nullable=False) content = db.Column(db.LargeBinary, nullable=False) file_format = db.Column(DBEnum(ImageMimeTypes), nullable=False) orientation = db.Column(DBEnum(Orientation), nullable=False) diff --git a/ereuse_devicehub/resources/lot/models.py b/ereuse_devicehub/resources/lot/models.py index 930288db..678f91bf 100644 --- a/ereuse_devicehub/resources/lot/models.py +++ b/ereuse_devicehub/resources/lot/models.py @@ -1,6 +1,7 @@ import uuid from datetime import datetime +from citext import CIText from flask import g from sqlalchemy import TEXT from sqlalchemy.dialects.postgresql import UUID @@ -11,13 +12,13 @@ from teal.db import UUIDLtree from ereuse_devicehub.db import db from ereuse_devicehub.resources.device.models import Device -from ereuse_devicehub.resources.models import STR_SIZE, Thing +from ereuse_devicehub.resources.models import Thing from ereuse_devicehub.resources.user.models import User class Lot(Thing): id = db.Column(UUID(as_uuid=True), primary_key=True) # uuid is generated on init by default - name = db.Column(db.Unicode(STR_SIZE), nullable=False) + name = db.Column(CIText(), nullable=False) closed = db.Column(db.Boolean, default=False, nullable=False) closed.comment = """ A closed lot cannot be modified anymore. diff --git a/ereuse_devicehub/resources/lot/schemas.py b/ereuse_devicehub/resources/lot/schemas.py index d0c723a3..9bfa0fd1 100644 --- a/ereuse_devicehub/resources/lot/schemas.py +++ b/ereuse_devicehub/resources/lot/schemas.py @@ -1,4 +1,5 @@ from marshmallow import fields as f +from teal.marshmallow import SanitizedStr from ereuse_devicehub.marshmallow import NestedOn from ereuse_devicehub.resources.device.schemas import Device @@ -9,7 +10,7 @@ from ereuse_devicehub.resources.schemas import Thing class Lot(Thing): id = f.UUID(dump_only=True) - name = f.String(validate=f.validate.Length(max=STR_SIZE), required=True) + name = SanitizedStr(validate=f.validate.Length(max=STR_SIZE), required=True) closed = f.Boolean(missing=False, description=m.Lot.closed.comment) devices = NestedOn(Device, many=True, dump_only=True) children = NestedOn('Lot', many=True, dump_only=True) diff --git a/ereuse_devicehub/resources/tag/__init__.py b/ereuse_devicehub/resources/tag/__init__.py index 5d9a1af5..8a01852f 100644 --- a/ereuse_devicehub/resources/tag/__init__.py +++ b/ereuse_devicehub/resources/tag/__init__.py @@ -3,7 +3,7 @@ import pathlib from click import argument, option from ereuse_utils import cli -from teal.resource import Resource +from teal.resource import Converters, Resource from teal.teal import Teal from ereuse_devicehub.db import db @@ -16,6 +16,7 @@ from ereuse_devicehub.resources.tag.view import TagDeviceView, TagView, get_devi class TagDef(Resource): SCHEMA = schema.Tag VIEW = TagView + ID_CONVERTER = Converters.lower ORG_H = 'The name of an existing organization in the DB. ' 'By default the organization operating this Devicehub.' diff --git a/ereuse_devicehub/resources/tag/model.py b/ereuse_devicehub/resources/tag/model.py index f7da86fb..465e073d 100644 --- a/ereuse_devicehub/resources/tag/model.py +++ b/ereuse_devicehub/resources/tag/model.py @@ -3,7 +3,7 @@ from contextlib import suppress from sqlalchemy import BigInteger, Column, ForeignKey, Unicode, UniqueConstraint from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import backref, relationship, validates -from teal.db import DB_CASCADE_SET_NULL, Query, URL +from teal.db import DB_CASCADE_SET_NULL, Query, URL, check_lower from teal.marshmallow import ValidationError from ereuse_devicehub.resources.agent.models import Organization @@ -12,7 +12,7 @@ from ereuse_devicehub.resources.models import Thing class Tag(Thing): - id = Column(Unicode(), primary_key=True) + id = Column(Unicode(), check_lower('id'), primary_key=True) id.comment = """The ID of the tag.""" org_id = Column(UUID(as_uuid=True), ForeignKey(Organization.id), @@ -37,7 +37,7 @@ class Tag(Thing): backref=backref('tags', lazy=True, collection_class=set), primaryjoin=Device.id == device_id) """The device linked to this tag.""" - secondary = Column(Unicode()) + secondary = Column(Unicode(), check_lower('secondary')) secondary.comment = """ A secondary identifier for this tag. It has the same constraints as the main one. Only needed in special cases. diff --git a/ereuse_devicehub/resources/tag/schema.py b/ereuse_devicehub/resources/tag/schema.py index ec6d0cb4..f41532df 100644 --- a/ereuse_devicehub/resources/tag/schema.py +++ b/ereuse_devicehub/resources/tag/schema.py @@ -1,6 +1,5 @@ -from marshmallow.fields import String from sqlalchemy.util import OrderedSet -from teal.marshmallow import URL +from teal.marshmallow import SanitizedStr, URL from ereuse_devicehub.marshmallow import NestedOn from ereuse_devicehub.resources.agent.schemas import Organization @@ -15,11 +14,12 @@ def without_slash(x: str) -> bool: class Tag(Thing): - id = String(description=m.Tag.id.comment, - validator=without_slash, - required=True) + id = SanitizedStr(lower=True, + description=m.Tag.id.comment, + validator=without_slash, + required=True) provider = URL(description=m.Tag.provider.comment, validator=without_slash) device = NestedOn(Device, dump_only=True) org = NestedOn(Organization, collection_class=OrderedSet, only_query='id') - secondary = String(description=m.Tag.secondary.comment) + secondary = SanitizedStr(lower=True, description=m.Tag.secondary.comment) diff --git a/ereuse_devicehub/resources/user/schemas.py b/ereuse_devicehub/resources/user/schemas.py index af47ff0e..e60ff1f8 100644 --- a/ereuse_devicehub/resources/user/schemas.py +++ b/ereuse_devicehub/resources/user/schemas.py @@ -2,6 +2,7 @@ from base64 import b64encode from marshmallow import post_dump from marshmallow.fields import Email, String, UUID +from teal.marshmallow import SanitizedStr from ereuse_devicehub.marshmallow import NestedOn from ereuse_devicehub.resources.agent.schemas import Individual @@ -11,9 +12,9 @@ from ereuse_devicehub.resources.schemas import Thing class User(Thing): id = UUID(dump_only=True) email = Email(required=True) - password = String(load_only=True, required=True) + password = SanitizedStr(load_only=True, required=True) individuals = NestedOn(Individual, many=True, dump_only=True) - name = String() + name = SanitizedStr() token = String(dump_only=True, description='Use this token in an Authorization header to access the app.' 'The token can change overtime.') diff --git a/examples/create-db.sh b/examples/create-db.sh index d178f005..b74fa7b0 100644 --- a/examples/create-db.sh +++ b/examples/create-db.sh @@ -6,3 +6,4 @@ psql -d $1 -c "CREATE USER dhub WITH PASSWORD 'ereuse';" # Create user Devicehub psql -d $1 -c "GRANT ALL PRIVILEGES ON DATABASE $1 TO dhub;" # Give access to the db psql -d $1 -c "CREATE EXTENSION pgcrypto SCHEMA public;" # Enable pgcrypto psql -d $1 -c "CREATE EXTENSION ltree SCHEMA public;" # Enable ltree +psql -d $1 -c "CREATE EXTENSION citext SCHEMA public;" # Enable citext diff --git a/requirements.txt b/requirements.txt index 2939b19b..b612d4e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ requests==2.19.1 requests-mock==1.5.2 SQLAlchemy==1.2.11 SQLAlchemy-Utils==0.33.3 -teal==0.2.0a19 +teal==0.2.0a20 webargs==4.0.0 Werkzeug==0.14.1 +sqlalchemy-citext==1.3.post0 diff --git a/setup.py b/setup.py index c78fcd43..83ae609c 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( long_description=long_description, long_description_content_type='text/markdown', install_requires=[ - 'teal>=0.2.0a19', # teal always first + 'teal>=0.2.0a20', # teal always first 'click', 'click-spinner', 'ereuse-rate==0.0.2', @@ -46,6 +46,7 @@ setup( 'PyYAML', 'requests', 'requests-toolbelt', + 'sqlalchemy-citext', 'sqlalchemy-utils[password, color, phone]', ], extras_require={ diff --git a/tests/conftest.py b/tests/conftest.py index bdf48fa1..7dee4a49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,7 @@ class TestConfig(DevicehubConfig): SCHEMA = 'test' TESTING = True ORGANIZATION_NAME = 'FooOrg' - ORGANIZATION_TAX_ID = 'FooOrgId' + ORGANIZATION_TAX_ID = 'foo-org-id' @pytest.fixture(scope='module') diff --git a/tests/test_agent.py b/tests/test_agent.py index a1561c6c..d379c7b5 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -18,7 +18,7 @@ from tests.conftest import app_context, create_user def test_agent(): """Tests creating an person.""" person = Person(name='Timmy', - tax_id='XYZ', + tax_id='xyz', country=Country.ES, telephone=PhoneNumber('+34666666666'), email='foo@bar.com') @@ -27,7 +27,7 @@ def test_agent(): p = schemas.Person().dump(person) assert p['name'] == person.name == 'Timmy' - assert p['taxId'] == person.tax_id == 'XYZ' + assert p['taxId'] == person.tax_id == 'xyz' assert p['country'] == person.country.name == 'ES' assert p['telephone'] == person.telephone.international == '+34 666 66 66 66' assert p['email'] == person.email == 'foo@bar.com' @@ -50,7 +50,7 @@ def test_system(): def test_organization(): """Tests creating an organization.""" org = Organization(name='ACME', - tax_id='XYZ', + tax_id='xyz', country=Country.ES, email='contact@acme.com') db.session.add(org) @@ -58,7 +58,7 @@ def test_organization(): o = schemas.Organization().dump(org) assert o['name'] == org.name == 'ACME' - assert o['taxId'] == org.tax_id == 'XYZ' + assert o['taxId'] == org.tax_id == 'xyz' assert org.country.name == o['country'] == 'ES' @@ -123,10 +123,10 @@ def test_assign_individual_user(): @pytest.mark.usefixtures(app_context.__name__) def test_create_organization_main_method(app: Devicehub): org_def = app.resources[models.Organization.t] # type: OrganizationDef - o = org_def.create_org('ACME', tax_id='FOO', country='ES') + o = org_def.create_org('ACME', tax_id='foo', country='ES') org = models.Agent.query.filter_by(id=o['id']).one() # type: Organization assert org.name == o['name'] == 'ACME' - assert org.tax_id == o['taxId'] == 'FOO' + assert org.tax_id == o['taxId'] == 'foo', 'FOO must be converted to lowercase' assert org.country.name == o['country'] == 'ES' diff --git a/tests/test_device.py b/tests/test_device.py index fdf45a00..fa1ff754 100644 --- a/tests/test_device.py +++ b/tests/test_device.py @@ -201,7 +201,7 @@ def test_sync_run_components_none(): @pytest.mark.usefixtures(conftest.app_context.__name__) -def test_sync_execute_register_Desktop_new_Desktop_no_tag(): +def test_sync_execute_register_desktop_new_Desktop_no_tag(): """ Syncs a new Desktop with HID and without a tag, creating it. :return: @@ -213,7 +213,7 @@ def test_sync_execute_register_Desktop_new_Desktop_no_tag(): @pytest.mark.usefixtures(conftest.app_context.__name__) -def test_sync_execute_register_Desktop_existing_no_tag(): +def test_sync_execute_register_desktop_existing_no_tag(): """ Syncs an existing Desktop with HID and without a tag. """ @@ -229,7 +229,7 @@ def test_sync_execute_register_Desktop_existing_no_tag(): @pytest.mark.usefixtures(conftest.app_context.__name__) -def test_sync_execute_register_Desktop_no_hid_no_tag(): +def test_sync_execute_register_desktop_no_hid_no_tag(): """ Syncs a Desktop without HID and no tag. @@ -243,18 +243,18 @@ def test_sync_execute_register_Desktop_no_hid_no_tag(): @pytest.mark.usefixtures(conftest.app_context.__name__) -def test_sync_execute_register_Desktop_tag_not_linked(): +def test_sync_execute_register_desktop_tag_not_linked(): """ Syncs a new Desktop with HID and a non-linked tag. It is OK if the tag was not linked, it will be linked in this process. """ - tag = Tag(id='FOO') + tag = Tag(id='foo') db.session.add(tag) db.session.commit() # Create a new transient non-db object - pc = Desktop(**conftest.file('pc-components.db')['device'], tags=OrderedSet([Tag(id='FOO')])) + pc = Desktop(**conftest.file('pc-components.db')['device'], tags=OrderedSet([Tag(id='foo')])) returned_pc = Sync().execute_register(pc) assert returned_pc == pc assert tag.device == pc, 'Tag has to be linked' diff --git a/tests/test_event.py b/tests/test_event.py index 1ef9fdb1..a116a148 100644 --- a/tests/test_event.py +++ b/tests/test_event.py @@ -89,7 +89,7 @@ def test_test_data_storage(): error=False, elapsed=timedelta(minutes=25), length=TestHardDriveLength.Short, - status='OK!', + status='ok!', lifetime=timedelta(days=120) ) db.session.add(test) @@ -199,13 +199,13 @@ def test_live(): db_live = models.Live(ip=ipaddress.ip_address('79.147.10.10'), subdivision_confidence=84, subdivision=Subdivision['ES-CA'], - city='Barcelona', + city='barcelona', city_confidence=20, - isp='ACME', + isp='acme', device=Desktop(serial_number='sn1', model='ml1', manufacturer='mr1', chassis=ComputerChassis.Docking), - organization='ACME1', - organization_type='ACME1bis') + organization='acme1', + organization_type='acme1bis') db.session.add(db_live) db.session.commit() client = UserClient(app, 'foo@foo.com', 'foo', response_wrapper=app.response_class) diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 3126c75a..8acf78c0 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -352,7 +352,7 @@ def assert_similar_device(device1: dict, device2: dict): assert isinstance(device1, dict) and device1 assert isinstance(device2, dict) and device2 for key in 'serialNumber', 'model', 'manufacturer', 'type': - assert device1.get(key, None) == device2.get(key, None) + assert device1.get(key, '').lower() == device2.get(key, '').lower() def assert_similar_components(components1: List[dict], components2: List[dict]): diff --git a/tests/test_tag.py b/tests/test_tag.py index 1305aaf9..e0044d18 100644 --- a/tests/test_tag.py +++ b/tests/test_tag.py @@ -21,7 +21,7 @@ from tests import conftest @pytest.mark.usefixtures(conftest.app_context.__name__) def test_create_tag(): """Creates a tag specifying a custom organization.""" - org = Organization(name='Bar', tax_id='BarTax') + org = Organization(name='bar', tax_id='bartax') tag = Tag(id='bar-1', org=org, provider=URL('http://foo.bar')) db.session.add(tag) db.session.commit() @@ -148,7 +148,7 @@ def test_tag_create_etags_cli(app: Devicehub, user: UserClient): catch_exceptions=False) with app.app_context(): tag = Tag.query.one() # type: Tag - assert tag.id == 'DT-BARBAR' + assert tag.id == 'dt-barbar' assert tag.secondary == 'foo' assert tag.provider == URL('https://t.ereuse.org') @@ -167,8 +167,13 @@ def test_tag_manual_link(app: Devicehub, user: UserClient): # Device already linked # Just returns an OK to conform to PUT as anything changes + user.put({}, res=Tag, item='foo-sec/device/{}'.format(desktop_id), status=204) + # Secondary IDs are case insensitive + user.put({}, res=Tag, item='FOO-BAR/device/{}'.format(desktop_id), status=204) + user.put({}, res=Tag, item='FOO-SEC/device/{}'.format(desktop_id), status=204) + # cannot link to another device when already linked user.put({}, res=Tag, item='foo-bar/device/99', status=LinkedToAnotherDevice) diff --git a/tests/test_workbench.py b/tests/test_workbench.py index d7abee31..116882fb 100644 --- a/tests/test_workbench.py +++ b/tests/test_workbench.py @@ -150,9 +150,9 @@ def test_real_eee_1001pxd(user: UserClient): pc, _ = user.get(res=Device, item=snapshot['device']['id']) assert pc['type'] == 'Laptop' assert pc['chassis'] == 'Netbook' - assert pc['model'] == '1001PXD' - assert pc['serialNumber'] == 'B8OAAS048286' - assert pc['manufacturer'] == 'ASUSTeK Computer INC.' + assert pc['model'] == '1001pxd' + assert pc['serialNumber'] == 'b8oaas048286' + assert pc['manufacturer'] == 'asustek computer inc.' assert pc['hid'] == 'asustek_computer_inc-b8oaas048286-1001pxd' assert pc['tags'] == [] components = snapshot['components'] @@ -170,7 +170,7 @@ def test_real_eee_1001pxd(user: UserClient): assert cpu['threads'] == 1 assert cpu['speed'] == 1.667 assert 'hid' not in cpu - assert cpu['model'] == 'Intel Atom CPU N455 @ 1.66GHz' + assert cpu['model'] == 'intel atom cpu n455 @ 1.66ghz' cpu, _ = user.get(res=Device, item=cpu['id']) events = cpu['events'] sysbench = next(e for e in events if e['type'] == em.BenchmarkProcessorSysbench.t) @@ -188,8 +188,8 @@ def test_real_eee_1001pxd(user: UserClient): assert em.Snapshot.t in event_types assert len(events) == 5 gpu = components[3] - assert gpu['model'] == 'Atom Processor D4xx/D5xx/N4xx/N5xx Integrated Graphics Controller' - assert gpu['manufacturer'] == 'Intel Corporation' + assert gpu['model'] == 'atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller' + assert gpu['manufacturer'] == 'intel corporation' assert gpu['memory'] == 256 gpu, _ = user.get(res=Device, item=gpu['id']) event_types = tuple(e['type'] for e in gpu['events']) @@ -198,9 +198,9 @@ def test_real_eee_1001pxd(user: UserClient): assert em.Snapshot.t in event_types assert len(event_types) == 3 sound = components[4] - assert sound['model'] == 'NM10/ICH7 Family High Definition Audio Controller' + assert sound['model'] == 'nm10/ich7 family high definition audio controller' sound = components[5] - assert sound['model'] == 'USB 2.0 UVC VGA WebCam' + assert sound['model'] == 'usb 2.0 uvc vga webcam' ram = components[6] assert ram['interface'] == 'DDR2' assert ram['speed'] == 667