trustchain-oc1-orchestral/devicehub-teal
Archived
10
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Delete lots; add LotDeviceDescendants view really fixing querying devices in lots

Browse source
This commit is contained in:
Xavier Bustamante Talavera 2018-11-11 21:52:55 +01:00
parent bf2c61ad65
commit bcf59de383
11 changed files with 188 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
ereuse_devicehub/db.py
View file

@ -1,4 +1,6 @@
from sqlalchemy import event
from sqlalchemy.dialects import postgresql
from sqlalchemy_utils import view
from teal.db import SchemaSQLAlchemy
@ -17,3 +19,21 @@ class SQLAlchemy(SchemaSQLAlchemy):
db = SQLAlchemy(session_options={"autoflush": False})
def create_view(name, selectable):
"""Creates a view.
This is an adaptation from sqlalchemy_utils.view. See
`the test on sqlalchemy-utils <https://github.com/kvesteri/
sqlalchemy-utils/blob/master/tests/test_views.py>`_ for an
example on how to use.
"""
table = view.create_table_from_selectable(name=name, selectable=selectable, metadata=None)
# We need to ensure views are created / destroyed before / after
# SchemaSQLAlchemy's listeners execute
# That is why insert=True in 'after_create'
event.listen(db.metadata, 'after_create', view.CreateView(name, selectable), insert=True)
event.listen(db.metadata, 'before_drop', view.DropView(name))
return table

6
ereuse_devicehub/resources/device/models.py
View file

@ -16,8 +16,8 @@ from sqlalchemy.orm import ColumnProperty, backref, relationship, validates
from sqlalchemy.util import OrderedSet
from sqlalchemy_utils import ColorType
from stdnum import imei, meid
from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, check_lower, \
check_range
from teal.db import CASCADE_DEL, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, \
check_lower, check_range
from teal.enums import Layouts
from teal.marshmallow import ValidationError
from teal.resource import url_for_resource
@ -428,7 +428,7 @@ class Component(Device):
parent = relationship(Computer,
backref=backref('components',
lazy=True,
cascade=CASCADE,
cascade=CASCADE_DEL,
order_by=lambda: Component.id,
collection_class=OrderedSet),
primaryjoin=parent_id == Computer.id)

13
ereuse_devicehub/resources/device/views.py
View file

@ -12,10 +12,10 @@ from teal.resource import View
from ereuse_devicehub import auth
from ereuse_devicehub.db import db
from ereuse_devicehub.resources import search
from ereuse_devicehub.resources.device.models import Component, Computer, Device, Manufacturer
from ereuse_devicehub.resources.device.models import Device, Manufacturer
from ereuse_devicehub.resources.device.search import DeviceSearch
from ereuse_devicehub.resources.event.models import Rate
from ereuse_devicehub.resources.lot.models import Lot, LotDevice
from ereuse_devicehub.resources.lot.models import LotDeviceDescendants
from ereuse_devicehub.resources.tag.model import Tag
@ -41,15 +41,10 @@ class TagQ(query.Query):
class LotQ(query.Query):
id = query.Or(query.QueryField(Lot.descendantsq, fields.UUID()))
id = query.Or(query.Equal(LotDeviceDescendants.ancestor_lot_id, fields.UUID()))
class Filters(query.Query):
_parent = Computer.__table__.alias()
_device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
_parent_device_in_lot = (Device.id == Component.id) & (Component.parent_id == _parent.c.id) \
& (_parent.c.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
type = query.Or(OfType(Device.type))
model = query.ILike(Device.model)
manufacturer = query.ILike(Device.manufacturer)
@ -59,7 +54,7 @@ class Filters(query.Query):
# todo This part of the query is really slow
# And forces usage of distinct, as it returns many rows
# due to having multiple paths to the same
lot = query.Join(_device_inside_lot | _parent_device_in_lot, LotQ)
lot = query.Join(Device.id == LotDeviceDescendants.device_id, LotQ)
class Sorting(query.Sort):

4
ereuse_devicehub/resources/event/models.py
View file

@ -18,7 +18,7 @@ from sqlalchemy.ext.orderinglist import ordering_list
from sqlalchemy.orm import backref, relationship, validates
from sqlalchemy.orm.events import AttributeEvents as Events
from sqlalchemy.util import OrderedSet
from teal.db import ArrayOfEnum, CASCADE, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \
from teal.db import ArrayOfEnum, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \
POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range
from teal.enums import Country, Currency, Subdivision
from teal.marshmallow import ValidationError
@ -219,7 +219,7 @@ class EventWithOneDevice(JoinedTableMixin, Event):
device = relationship(Device,
backref=backref('events_one',
lazy=True,
cascade=CASCADE,
cascade=CASCADE_OWN,
order_by=lambda: EventWithOneDevice.created,
collection_class=OrderedSet),
primaryjoin=Device.id == device_id)

95
ereuse_devicehub/resources/lot/models.py
View file

@ -1,5 +1,6 @@
import uuid
from datetime import datetime
from typing import Union
from boltons import urlutils
from citext import CIText
@ -9,11 +10,11 @@ from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.sql import expression as exp
from sqlalchemy_utils import LtreeType
from sqlalchemy_utils.types.ltree import LQUERY
from teal.db import UUIDLtree
from teal.db import CASCADE_OWN, UUIDLtree
from teal.resource import url_for_resource
from ereuse_devicehub.db import db
from ereuse_devicehub.resources.device.models import Device
from ereuse_devicehub.db import create_view, db
from ereuse_devicehub.resources.device.models import Component, Computer, Device
from ereuse_devicehub.resources.models import Thing
from ereuse_devicehub.resources.user.models import User
@ -89,6 +90,16 @@ class Lot(Thing):
_id = UUIDLtree.convert(id)
return (cls.id == Path.lot_id) & Path.path.lquery(exp.cast('*.{}.*'.format(_id), LQUERY))
@classmethod
def device_in_lotq(cls):
parent = Computer.__table__.alias()
device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
parent_device_in_lot = (Device.id == Component.id) \
& (Component.parent_id == parent.c.id) \
& (parent.c.id == LotDevice.device_id) \
& (Lot.id == LotDevice.lot_id)
return device_inside_lot | parent_device_in_lot
@property
def parents(self):
return self.parentsq(self.id)
@ -109,8 +120,28 @@ class Lot(Thing):
"""Gets the lots that are not under any other lot."""
return cls.query.join(cls.paths).filter(db.func.nlevel(Path.path) == 1)
def __contains__(self, child: 'Lot'):
def delete(self):
"""Deletes the lot.
This method removes the children lots and children
devices orphan from this lot and then marks this lot
for deletion.
"""
for child in self.children:
self.remove_child(child)
db.session.delete(self)
def __contains__(self, child: Union['Lot', Device]):
if isinstance(child, Lot):
return Path.has_lot(self.id, child.id)
elif isinstance(child, Device):
device = db.session.query(LotDeviceDescendants) \
.filter(LotDeviceDescendants.device_id == child.id) \
.filter(LotDeviceDescendants.ancestor_lot_id == self.id) \
.one_or_none()
return device
else:
raise TypeError('Lot only contains devices and lots, not {}'.format(child.__class__))
def __repr__(self) -> str:
return '<Lot {0.name} devices={0.devices!r}>'.format(self)
@ -136,7 +167,10 @@ class Path(db.Model):
server_default=db.text('gen_random_uuid()'))
lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False, index=True)
lot = db.relationship(Lot,
backref=db.backref('paths', lazy=True, collection_class=set),
backref=db.backref('paths',
lazy=True,
collection_class=set,
cascade=CASCADE_OWN),
primaryjoin=Lot.id == lot_id)
path = db.Column(LtreeType, nullable=False)
created = db.Column(db.TIMESTAMP(timezone=True), server_default=db.text('CURRENT_TIMESTAMP'))
@ -174,3 +208,54 @@ class Path(db.Model):
"SELECT 1 from path where path ~ '*.{}.*.{}.*'".format(parent_id, child_id)
).first()
)
class LotDeviceDescendants(db.Model):
"""A view facilitating querying inclusion between devices and lots,
including components.
The view has 4 columns:
1. The ID of the device.
2. The ID of a lot containing the device.
3. The ID of the lot that directly contains the device.
4. If 1. is a component, the ID of the device that is inside the lot.
"""
_ancestor = Lot.__table__.alias(name='ancestor')
"""Ancestor lot table."""
_desc = Lot.__table__.alias()
"""Descendant lot table."""
lot_device = _desc \
.join(LotDevice, _desc.c.id == LotDevice.lot_id) \
.join(Path, _desc.c.id == Path.lot_id)
"""Join: Path -- Lot -- LotDevice"""
descendants = "path.path ~ (CAST('*.'|| replace(CAST({}.id as text), '-', '_') " \
"|| '.*' AS LQUERY))".format(_ancestor.name)
"""Query that gets the descendants of the ancestor lot."""
devices = db.select([
LotDevice.device_id,
_ancestor.c.id.label('ancestor_lot_id'),
_desc.c.id.label('parent_lot_id'),
None
]).select_from(_ancestor).select_from(lot_device).where(descendants)
# Components
_parent_device = Device.__table__.alias(name='parent_device')
"""The device that has the access to the lot."""
lot_device_component = lot_device \
.join(_parent_device, _parent_device.c.id == LotDevice.device_id) \
.join(Component, _parent_device.c.id == Component.parent_id)
"""Join: Path -- Lot -- LotDevice -- ParentDevice (Device) -- Component"""
components = db.select([
Component.id.label('device_id'),
_ancestor.c.id.label('ancestor_lot_id'),
_desc.c.id.label('parent_lot_id'),
LotDevice.device_id.label('device_parent_id'),
]).select_from(_ancestor).select_from(lot_device_component).where(descendants)
__table__ = create_view(
name='lot_device_descendants',
selectable=devices.union(components)
)

20
ereuse_devicehub/resources/lot/models.pyi
View file

@ -1,6 +1,6 @@
import uuid
from datetime import datetime
from typing import Iterable, Set, Union
from typing import Iterable, Optional, Set, Union
from uuid import UUID
from boltons import urlutils
@ -8,6 +8,7 @@ from sqlalchemy import Column
from sqlalchemy.orm import Query, relationship
from sqlalchemy_utils import Ltree
from ereuse_devicehub.db import db
from ereuse_devicehub.resources.device.models import Device
from ereuse_devicehub.resources.models import Thing
@ -65,6 +66,9 @@ class Lot(Thing):
def url(self) -> urlutils.URL:
pass
def delete(self):
pass
class Path:
id = ... # type: Column
@ -79,3 +83,17 @@ class Path:
self.lot = ... # type: Lot
self.path = ... # type: Ltree
self.created = ... # type: datetime
class LotDeviceDescendants(db.Model):
device_id = ... # type: Column
ancestor_lot_id = ... # type: Column
parent_lot_id = ... # type: Column
device_parent_id = ... # type: Column
def __init__(self) -> None:
super().__init__()
self.device_id = ... # type: int
self.ancestor_lot_id = ... # type: UUID
self.parent_lot_id = ... # type: UUID
self.device_parent_id = ... # type: Optional[int]

6
ereuse_devicehub/resources/lot/views.py
View file

@ -97,6 +97,12 @@ class LotView(View):
cls._p(nodes, path)
return nodes
def delete(self, id):
lot = Lot.query.filter_by(id=id).one()
lot.delete()
db.session.commit()
return Response(status=204)
@classmethod
def _p(cls, nodes: List[dict], path: deque):
"""Recursively creates the nested lot structure.

3
ereuse_devicehub/resources/models.pyi
View file

@ -1,6 +1,6 @@
from datetime import datetime
from sqlalchemy import Column
from sqlalchemy import Column, Table
from teal.db import Model
STR_SIZE = 64
@ -10,6 +10,7 @@ STR_XSM_SIZE = 16
class Thing(Model):
__table__ = ... # type: Table
t = ... # type: str
type = ... # type: str
updated = ... # type: Column

6
requirements.txt
View file

@ -23,9 +23,9 @@ python-stdnum==1.9
PyYAML==3.13
requests==2.19.1
requests-mock==1.5.2
SQLAlchemy==1.2.11
SQLAlchemy-Utils==0.33.3
teal==0.2.0a29
SQLAlchemy==1.2.14
SQLAlchemy-Utils==0.33.6
teal==0.2.0a30
webargs==4.0.0
Werkzeug==0.14.1
sqlalchemy-citext==1.3.post0

2
setup.py
View file

@ -29,7 +29,7 @@ setup(
long_description=long_description,
long_description_content_type='text/markdown',
install_requires=[
'teal>=0.2.0a29', # teal always first
'teal>=0.2.0a30', # teal always first
'click',
'click-spinner',
'ereuse-utils[Naming]>=0.4b10',

59
tests/test_lot.py
View file

@ -23,7 +23,40 @@ In case of error, debug with:
"""
def test_lot_modify_patch_endpoint(user: UserClient):
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_lot_model_children():
"""Tests the property Lot.children
l1
|
l2
|
l3
"""
lots = Lot('1'), Lot('2'), Lot('3')
l1, l2, l3 = lots
db.session.add_all(lots)
db.session.flush()
l1.add_child(l2)
db.session.flush()
assert list(l1.children) == [l2]
l2.add_child(l3)
assert list(l1.children) == [l2]
l2.delete()
db.session.flush()
assert not list(l1.children)
l1.delete()
db.session.flush()
l3b = Lot.query.one()
assert l3 == l3b
def test_lot_modify_patch_endpoint_and_delete(user: UserClient):
"""Creates and modifies lot properties through the endpoint"""
l, _ = user.post({'name': 'foo', 'description': 'baz'}, res=Lot)
assert l['name'] == 'foo'
@ -32,20 +65,17 @@ def test_lot_modify_patch_endpoint(user: UserClient):
l_after, _ = user.get(res=Lot, item=l['id'])
assert l_after['name'] == 'bar'
assert l_after['description'] == 'bax'
user.delete(res=Lot, item=l['id'], status=204)
user.get(res=Lot, item=l['id'], status=404)
@pytest.mark.xfail(reason='No DEL endpoint')
def test_lot_delete_endpoint(user: UserClient):
pass
@pytest.mark.xfail(reason='the IN comparison does not work for device')
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_lot_device_relationship():
device = Desktop(serial_number='foo',
model='bar',
manufacturer='foobar',
chassis=ComputerChassis.Lunchbox)
device.components.add(GraphicCard(serial_number='foo', model='bar1', manufacturer='baz'))
child = Lot('child')
child.devices.add(device)
db.session.add(child)
@ -253,21 +283,6 @@ def test_lot_roots():
assert set(Lot.roots()) == {l1, l3}
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_lot_model_children():
"""Tests the property Lot.children"""
lots = Lot('1'), Lot('2'), Lot('3')
l1, l2, l3 = lots
db.session.add_all(lots)
db.session.flush()
l1.add_child(l2)
db.session.flush()
children = l1.children
assert list(children) == [l2]
def test_post_get_lot(user: UserClient):
"""Tests submitting and retreiving a basic lot."""
l, _ = user.post({'name': 'Foo'}, res=Lot)