trustchain-oc1-orchestral
/
devicehub-teal
Archived
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Draft on lot

This commit is contained in:
Xavier Bustamante Talavera 2018-08-08 21:25:53 +02:00
parent bc433d80e9
commit f5d69070e6
14 changed files with 360 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -22,7 +22,7 @@ Devicehub is built with [Teal](https://github.com/bustawin/teal) and
The requirements are:
- Python 3.5.3 or higher. In debian 9 is `# apt install python3-pip`.
- PostgreSQL 9.6 or higher. In debian 9 is `# apt install postgresql`
- PostgreSQL 9.6 or higher. In debian 9 is `# apt install postgresql-contrib`
- passlib. In debian 9 is `# apt install python3-passlib`.
Install Devicehub with *pip*: `pip3 install ereuse-devicehub -U --pre`.
@ -39,6 +39,8 @@ postgres $ createdb devicehub # Create main database
postgres $ psql devicehub # Access to the database
postgres $ CREATE USER dhub WITH PASSWORD 'ereuse'; # Create user devicehub uses to access db
postgres $ GRANT ALL PRIVILEGES ON DATABASE devicehub TO dhub; # Give access to the db
postgres $ CREATE EXTENSION pgcrypto SCHEMA public; # Enable pgcrypto
postgres $ CREATE EXTENSION ltree SCHEMA public; # Enable ltree
postgres $ \q
exit
```

5
ereuse_devicehub/config.py
View File

@ -2,7 +2,7 @@ from distutils.version import StrictVersion
from itertools import chain
from typing import Set
from ereuse_devicehub.resources import agent, device, event, inventory, tag, user
from ereuse_devicehub.resources import agent, device, event, inventory, lot, tag, user
from ereuse_devicehub.resources.enums import PriceSoftware, RatingSoftware
from teal.auth import TokenAuth
from teal.config import Config
@ -16,7 +16,8 @@ class DevicehubConfig(Config):
import_resource(user),
import_resource(tag),
import_resource(inventory),
import_resource(agent)))
import_resource(agent),
import_resource(lot)))
PASSWORD_SCHEMES = {'pbkdf2_sha256'} # type: Set[str]
SQLALCHEMY_DATABASE_URI = 'postgresql://dhub:ereuse@localhost/devicehub' # type: str
SCHEMA = 'dhub'

9
ereuse_devicehub/db.py
View File

@ -1,7 +1,16 @@
from sqlalchemy.dialects import postgresql
from teal.db import SQLAlchemy as _SQLAlchemy
class SQLAlchemy(_SQLAlchemy):
"""
Superuser must create the required extensions in the public
schema of the database, as it is in the `search_path`
defined in teal.
"""
UUID = postgresql.UUID
def drop_all(self, bind='__all__', app=None):
"""A faster nuke-like option to drop everything."""
self.drop_schema()

6
ereuse_devicehub/resources/event/models.py
View File

@ -51,7 +51,7 @@ class Event(Thing):
incidence.comment = """
Should this event be reviewed due some anomaly?
"""
closed = Column(Boolean, default=True, nullable=False)
closed = Column(Boolean, default=False, nullable=False)
closed.comment = """
Whether the author has finished the event.
After this is set to True, no modifications are allowed.
@ -100,7 +100,7 @@ class Event(Thing):
author = relationship(User,
backref=backref('authored_events', lazy=True, collection_class=set),
primaryjoin=author_id == User.id)
"""
author_id.comment = """
The user that recorded this action in the system.
This does not necessarily has to be the person that produced
@ -118,7 +118,7 @@ class Event(Thing):
lazy=True,
collection_class=OrderedSet,
order_by=lambda: Event.created),
primaryjoin=agent_id == Agent.id, )
primaryjoin=agent_id == Agent.id)
agent_id.comment = """
The direct performer or driver of the action. e.g. John wrote a book.

2
ereuse_devicehub/resources/event/schemas.py
View File

@ -24,7 +24,7 @@ from teal.resource import Schema
class Event(Thing):
id = UUID(dump_only=True)
name = String(default='', validate=Length(STR_BIG_SIZE), description=m.Event.name.comment)
name = String(default='', validate=Length(max=STR_BIG_SIZE), description=m.Event.name.comment)
incidence = Boolean(default=False, description=m.Event.incidence.comment)
closed = Boolean(missing=True, description=m.Event.closed.comment)
error = Boolean(default=False, description=m.Event.error.comment)

19
ereuse_devicehub/resources/lot/__init__.py Normal file
View File

@ -0,0 +1,19 @@
import pathlib
from ereuse_devicehub.db import db
from ereuse_devicehub.resources.lot import schemas
from ereuse_devicehub.resources.lot.views import LotView
from teal.resource import Converters, Resource
class LotDef(Resource):
SCHEMA = schemas.Lot
VIEW = LotView
AUTH = True
ID_CONVERTER = Converters.uuid
def init_db(self, db: 'db.SQLAlchemy'):
# Create functions
with pathlib.Path(__file__).parent.joinpath('dag.sql').open() as f:
sql = f.read()
db.session.execute(sql)

110
ereuse_devicehub/resources/lot/dag.sql Normal file
View File

@ -0,0 +1,110 @@
CREATE OR REPLACE FUNCTION add_edge(parent_id uuid, child_id uuid)
/* Adds an edge between ``parent`` and ``child``.
Designed to work with Directed Acyclic Graphs (DAG)
(or said in another way, trees with multiple parents without cycles).
This method will raise an exception if:
- Parent is the same as child.
- Child contains the parent.
- Edge parent - child already exists.
Influenced by:
- https://www.codeproject.com/Articles/22824/A-Model-to-Represent-Directed-Acyclic-Graphs-DAG
- http://patshaughnessy.net/2017/12/12/installing-the-postgres-ltree-extension
- https://en.wikipedia.org/wiki/Directed_acyclic_graph
*/
RETURNS void AS $$
DECLARE
parent text := replace(CAST(parent_id as text), '-', '_');
child text := replace(CAST(child_id as text), '-', '_');
BEGIN
if parent = child
then
raise exception 'Cannot create edge: the parent is the same as the child.';
end if;
if exists(
select 1 from edge where edge.path ~ CAST('*.' || child || '.*.' || parent || '.*' as lquery)
)
then
raise exception 'Cannot create edge: child already contains parent.';
end if;
-- We have two subgraphs: the parent subgraph that goes from the parent to the root,
-- and the child subgraph, going from the child (which is the root of this subgraph)
-- to all the leafs.
-- We do the cartesian product from all the paths of the parent subgraph that end in the parent
-- WITH all the paths that start from the child that end to its leafs.
insert into edge (lot_id, path) (select distinct lot_id, fp.path ||
subpath(edge.path, index(edge.path, text2ltree(child)))
from edge,
(select path
from edge
where path ~ CAST('*.' || parent AS lquery)) as fp
where edge.path ~ CAST('*.' || child || '.*' AS lquery));
-- Cleanup: old paths that start with the child (that where used above in the cartesian product)
-- have became a subset of the result of the cartesian product, thus being redundant.
delete from edge where edge.path ~ CAST(child || '.*' AS lquery);
END
$$
LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_edge(parent_id uuid, child_id uuid)
/* Deletes an edge between ``parent`` and ``child``.
Designed to work with DAG (See ``add_edge`` function).
This method will raise an exception if the relationship does not
exist.
*/
RETURNS void AS $$
DECLARE
parent text := replace(CAST(parent_id as text), '-', '_');
child text := replace(CAST(child_id as text), '-', '_');
number int;
BEGIN
-- to delete we remove from the path of the descendants of the child
-- (and the child) any ancestor coming from this edge.
-- When we added the edge we did a cartesian product. When removing
-- this part of the path we will have duplicate paths.
-- don't check uniqueness for path key until we delete duplicates
SET CONSTRAINTS edge_path_unique DEFERRED;
-- remove everything above the child lot_id in the path
-- this creates duplicates on path and lot_id
update edge
set path = subpath(path, index(path, text2ltree(child)))
where path ~ CAST('*.' || parent || '.' || child || '.*' AS lquery);
-- remove duplicates
-- we need an id field exclusively for this operation
-- from https://wiki.postgresql.org/wiki/Deleting_duplicates
DELETE
FROM edge
WHERE id IN (SELECT id
FROM (SELECT id, ROW_NUMBER() OVER (partition BY lot_id, path) AS rnum FROM edge) t
WHERE t.rnum > 1);
-- re-activate uniqueness check and perform check
SET CONSTRAINTS edge_path_unique IMMEDIATE;
-- After the update the one of the paths of the child will be
-- containing only the child.
-- This can only be when the child has no parent at all.
-- In case the child has more than one parent, remove this path
-- (note that we want it to remove it too from descendants of this
-- child, ex. 'child_id'.'desc1')
select COUNT(1) into number from edge where lot_id = child_id;
IF number > 1
THEN
delete from edge where path <@ text2ltree(child);
end if;
END
$$
LANGUAGE plpgsql;

101
ereuse_devicehub/resources/lot/models.py Normal file
View File

@ -0,0 +1,101 @@
import uuid
from datetime import datetime
from typing import Set
from flask import g
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.sql import expression
from sqlalchemy_utils import LtreeType
from sqlalchemy_utils.types.ltree import LQUERY
from ereuse_devicehub.db import db
from ereuse_devicehub.resources.device.models import Device
from ereuse_devicehub.resources.models import STR_SIZE, Thing
from ereuse_devicehub.resources.user.models import User
class Lot(Thing):
id = db.Column(UUID(as_uuid=True),
primary_key=True,
server_default=db.text('gen_random_uuid()'))
name = db.Column(db.Unicode(STR_SIZE), nullable=False)
closed = db.Column(db.Boolean, default=False, nullable=False)
closed.comment = """
A closed lot cannot be modified anymore.
"""
devices = db.relationship(Device,
backref=db.backref('parents', lazy=True, collection_class=set),
secondary=lambda: LotDevice.__table__,
collection_class=set)
def __repr__(self) -> str:
return '<Lot {0.name} devices={0.devices!r}>'.format(self)
def add_child(self, child: 'Lot'):
"""Adds a child to this lot."""
Edge.add(self.id, child.id)
db.session.refresh(self) # todo is this useful?
db.session.refresh(child)
def remove_child(self, child: 'Lot'):
Edge.delete(self.id, child.id)
def __contains__(self, child: 'Lot'):
return Edge.has_lot(self.id, child.id)
class LotDevice(db.Model):
device_id = db.Column(db.BigInteger, db.ForeignKey(Device.id), primary_key=True)
lot_id = db.Column(UUID(as_uuid=True), db.ForeignKey(Lot.id), primary_key=True)
created = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
author_id = db.Column(UUID(as_uuid=True),
db.ForeignKey(User.id),
nullable=False,
default=lambda: g.user.id)
author = db.relationship(User, primaryjoin=author_id == User.id)
author_id.comment = """
The user that put the device in the lot.
"""
class Edge(Thing):
id = db.Column(db.UUID(as_uuid=True),
primary_key=True,
server_default=db.text('gen_random_uuid()'))
lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False)
lot = db.relationship(Lot,
backref=db.backref('edges', lazy=True, collection_class=set),
primaryjoin=Lot.id == lot_id)
path = db.Column(LtreeType, unique=True, nullable=False)
__table_args__ = (
db.UniqueConstraint(path, name='edge_path_unique', deferrable=True, initially='immediate'),
db.Index('path_gist', path, postgresql_using='gist'),
db.Index('path_btree', path, postgresql_using='btree')
)
def children(self) -> Set['Edge']:
"""Get the children edges."""
# From https://stackoverflow.com/a/41158890
exp = '*.{}.*{{1}}'.format(self.lot_id)
return set(self.query
.filter(self.path.lquery(expression.cast(exp, LQUERY)))
.distinct(self.__class__.lot_id)
.all())
@classmethod
def add(cls, parent_id: uuid.UUID, child_id: uuid.UUID):
"""Creates an edge between parent and child."""
db.session.execute(db.func.add_edge(str(parent_id), str(child_id)))
@classmethod
def delete(cls, parent_id: uuid.UUID, child_id: uuid.UUID):
"""Deletes the edge between parent and child."""
db.session.execute(db.func.delete_edge(str(parent_id), str(child_id)))
@classmethod
def has_lot(cls, parent_id: uuid.UUID, child_id: uuid.UUID) -> bool:
return bool(db.session.execute(
"SELECT 1 from edge where path ~ '*.{}.*.{}.*'".format(
str(parent_id).replace('-', '_'), str(child_id).replace('-', '_'))
).first())

14
ereuse_devicehub/resources/lot/schemas.py Normal file
View File

@ -0,0 +1,14 @@
from marshmallow import fields as f
from ereuse_devicehub.marshmallow import NestedOn
from ereuse_devicehub.resources.device.schemas import Device
from ereuse_devicehub.resources.lot import models as m
from ereuse_devicehub.resources.models import STR_SIZE
from ereuse_devicehub.resources.schemas import Thing
class Lot(Thing):
id = f.UUID(dump_only=True)
name = f.String(validate=f.validate.Length(max=STR_SIZE))
closed = f.String(required=True, missing=False, description=m.Lot.closed.comment)
devices = f.String(NestedOn(Device, many=True, collection_class=set, only_query='id'))

25
ereuse_devicehub/resources/lot/views.py Normal file
View File

@ -0,0 +1,25 @@
import uuid
from flask import current_app as app, request
from ereuse_devicehub.db import db
from ereuse_devicehub.resources.lot.models import Lot
from teal.resource import View
class LotView(View):
def post(self):
json = request.get_json(validate=False)
e = app.resources[json['type']].schema.load(json)
Model = db.Model._decl_class_registry.data[json['type']]()
lot = Model(**e)
db.session.add(lot)
db.session.commit()
ret = self.schema.jsonify(lot)
ret.status_code = 201
return ret
def one(self, id: uuid.UUID):
"""Gets one event."""
event = Lot.query.filter_by(id=id).one()
return self.schema.jsonify(event)

2
ereuse_devicehub/resources/tag/model.py
View File

@ -38,7 +38,7 @@ class Tag(Thing):
return value
__table_args__ = (
UniqueConstraint(device_id, org_id, name='One tag per organization.'),
UniqueConstraint(device_id, org_id, name='one_tag_per_organization'),
)
def __repr__(self) -> str:

5
tests/test_basic.py
View File

@ -24,7 +24,8 @@ def test_api_docs(client: Client):
'/tags/',
'/snapshots/',
'/users/login',
'/events/'
'/events/',
'/lots/'
}
assert docs['info'] == {'title': 'Devicehub', 'version': '0.2'}
assert docs['components']['securitySchemes']['bearerAuth'] == {
@ -35,4 +36,4 @@ def test_api_docs(client: Client):
'scheme': 'basic',
'name': 'Authorization'
}
assert 76 == len(docs['definitions'])
assert 77 == len(docs['definitions'])

65
tests/test_lot.py Normal file
View File

@ -0,0 +1,65 @@
import pytest
from flask import g
from sqlalchemy_utils import Ltree
from ereuse_devicehub.db import db
from ereuse_devicehub.resources.device.models import Desktop
from ereuse_devicehub.resources.enums import ComputerChassis
from ereuse_devicehub.resources.lot.models import Edge, Lot, LotDevice
from tests import conftest
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_lot_device_relationship():
device = Desktop(serial_number='foo',
model='bar',
manufacturer='foobar',
chassis=ComputerChassis.Lunchbox)
lot = Lot(name='lot1')
lot.devices.add(device)
db.session.add(lot)
db.session.flush()
lot_device = LotDevice.query.one() # type: LotDevice
assert lot_device.device_id == device.id
assert lot_device.lot_id == lot.id
assert lot_device.created
assert lot_device.author_id == g.user.id
assert device.parents == {lot}
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_add_edge():
child = Lot(name='child')
parent = Lot(name='parent')
db.session.add(child)
db.session.add(parent)
db.session.flush()
# todo edges should automatically be created when the lot is created
child.edges.add(Edge(path=Ltree(str(child.id).replace('-', '_'))))
parent.edges.add(Edge(path=Ltree(str(parent.id).replace('-', '_'))))
db.session.flush()
parent.add_child(child)
assert child in parent
assert len(child.edges) == 1
assert len(parent.edges) == 1
parent.remove_child(child)
assert child not in parent
assert len(child.edges) == 1
assert len(parent.edges) == 1
grandparent = Lot(name='grandparent')
db.session.add(grandparent)
db.session.flush()
grandparent.edges.add(Edge(path=Ltree(str(grandparent.id).replace('-', '_'))))
db.session.flush()
grandparent.add_child(parent)
parent.add_child(child)
assert parent in grandparent
assert child in parent
assert child in grandparent

5
tests/test_rate.py
View File

@ -8,9 +8,10 @@ from ereuse_devicehub.resources.enums import Bios, ComputerChassis, ImageMimeTyp
RatingSoftware
from ereuse_devicehub.resources.event.models import PhotoboxRate, WorkbenchRate
from ereuse_devicehub.resources.image.models import Image, ImageList
from tests import conftest
@pytest.mark.usefixtures('auth_app_context')
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_workbench_rate_db():
rate = WorkbenchRate(processor=0.1,
ram=1.0,
@ -25,7 +26,7 @@ def test_workbench_rate_db():
db.session.commit()
@pytest.mark.usefixtures('auth_app_context')
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_photobox_rate_db():
pc = Desktop(serial_number='24', chassis=ComputerChassis.Tower)
image = Image(name='foo',