pedro/django-orchestra-test
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Replace cached, IPy and cmp_key for python3 standard library

Browse source
This commit is contained in:
Marc Aymerich 2015-10-02 11:08:23 +00:00
parent b5a13af46c
commit 4173d7de27
25 changed files with 120 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
TODO.md
View file

@ -417,8 +417,5 @@ mkhomedir_helper or create ssh homes with bash.rc and such
# validate saas setting allow_custom_url check that websites have a related declared directive
# warnings if some plugins are disabled, like make routes red
# replace cached by https://docs.python.org/3/library/functools.html#functools.lru_cache
# replace IPy by https://docs.python.org/3/library/ipaddress.html#module-ipaddress
# replace show emails by https://docs.python.org/3/library/email.contentmanager.html#module-email.contentmanager
# https://docs.python.org/3/library/functools.html#functools.cmp_to_key
# tzinfo=datetime.timezone.utc

12
orchestra/contrib/bills/models.py
View file

@ -1,5 +1,6 @@
import datetime
from dateutil.relativedelta import relativedelta
from functools import lru_cache
from django.core.validators import ValidationError, RegexValidator
from django.db import models
@ -14,7 +15,6 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.accounts.models import Account
from orchestra.contrib.contacts.models import Contact
from orchestra.core import validators
from orchestra.utils.functional import cached
from orchestra.utils.html import html_to_pdf
from . import settings
@ -319,7 +319,7 @@ class Bill(models.Model):
self.number = self.get_number()
super(Bill, self).save(*args, **kwargs)
@cached
@lru_cache()
def compute_subtotals(self):
subtotals = {}
lines = self.lines.annotate(totals=F('subtotal') + Sum(Coalesce('sublines__total', 0)))
@ -333,21 +333,21 @@ class Bill(models.Model):
result[tax] = (subtotal, round(tax/100*subtotal, 2))
return result
@cached
@lru_cache()
def compute_base(self):
bases = self.lines.annotate(
bases=F('subtotal') + Sum(Coalesce('sublines__total', 0))
)
return round(bases.aggregate(Sum('bases'))['bases__sum'] or 0, 2)
@cached
@lru_cache()
def compute_tax(self):
taxes = self.lines.annotate(
taxes=(F('subtotal') + Coalesce(Sum('sublines__total'), 0)) * (F('tax')/100)
)
return round(taxes.aggregate(Sum('taxes'))['taxes__sum'] or 0, 2)
@cached
@lru_cache()
def compute_total(self):
if 'lines' in getattr(self, '_prefetched_objects_cache', ()):
total = 0
@ -430,7 +430,7 @@ class BillLine(models.Model):
return ini
return "{ini} / {end}".format(ini=ini, end=end)
@cached
@lru_cache()
def compute_total(self):
total = self.subtotal or 0
if hasattr(self, 'subline_total'):

4
orchestra/contrib/payments/methods/options.py
View file

@ -1,8 +1,8 @@
import logging
from dateutil import relativedelta
from functools import lru_cache
from orchestra import plugins
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class
from .. import settings
@ -20,7 +20,7 @@ class PaymentMethod(plugins.Plugin):
state_help = {}
@classmethod
@cached
@lru_cache()
def get_plugins(cls):
plugins = []
for cls in settings.PAYMENTS_ENABLED_METHODS:

7
orchestra/contrib/plans/models.py
View file

@ -1,3 +1,5 @@
from functools import lru_cache
from django.core.validators import ValidationError
from django.db import models
from django.db.models import Q
@ -6,7 +8,6 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.core.validators import validate_name
from orchestra.models import queryset
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class
from . import settings
@ -84,12 +85,12 @@ class Rate(models.Model):
return "{}-{}".format(str(self.price), self.quantity)
@classmethod
@cached
@lru_cache()
def get_methods(cls):
return dict((method, import_class(method)) for method in settings.PLANS_RATE_METHODS)
@classmethod
@cached
@lru_cache()
def get_choices(cls):
choices = []
for name, method in cls.get_methods().items():

4
orchestra/contrib/resources/admin.py
View file

@ -1,3 +1,4 @@
from functools import lru_cache
from urllib.parse import parse_qs
from django.apps import apps
@ -19,7 +20,6 @@ from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_
from orchestra.contrib.orchestration.models import Route
from orchestra.core import services
from orchestra.utils import db, sys
from orchestra.utils.functional import cached
from .actions import run_monitor, show_history
from .api import history_data
@ -240,7 +240,7 @@ def resource_inline_factory(resources):
def total_form_count(self, resources=resources):
return len(resources)
@cached
@lru_cache()
def get_queryset(self):
""" Filter disabled resources """
queryset = super(ResourceInlineFormSet, self).get_queryset()

20
orchestra/contrib/saas/backends/moodle.py
View file

@ -33,7 +33,8 @@ class MoodleMuBackend(ServiceController):
$wwwroot = "https://{$site}-courses.pangea.org";
}
}
$CFG->prefix = "${site}_";
$prefix = str_replace('-', '_', $site);
$CFG->prefix = "${prefix}_";
$CFG->wwwroot = $wwwroot;
$CFG->dataroot = "/home/pangea/moodledata/{$site}/";
"""
@ -43,6 +44,9 @@ class MoodleMuBackend(ServiceController):
def save(self, webapp):
context = self.get_context(webapp)
self.delete_site_map(context)
if context['custom_url']:
self.insert_site_map(context)
self.append(textwrap.dedent("""\
mkdir -p %(moodledata_path)s
chown %(user)s:%(user)s %(moodledata_path)s
@ -64,9 +68,9 @@ class MoodleMuBackend(ServiceController):
--host="%(db_host)s" \\
--user="%(db_user)s" \\
--password="%(db_pass)s" \\
--execute='UPDATE %(site_name)s_user
--execute='UPDATE %(db_prefix)s_user
SET password=MD5("%(password)s")
WHERE username="admin"' \\
WHERE username="admin";' \\
%(db_name)s
""") % context
)
@ -83,9 +87,6 @@ class MoodleMuBackend(ServiceController):
EOF
fi""") % context
)
self.delete_site_map(context)
if context['custom_url']:
self.insert_site_map(context)
def delete_site_map(self, context):
self.append(textwrap.dedent("""\
@ -105,7 +106,7 @@ class MoodleMuBackend(ServiceController):
context = self.get_context(saas)
self.append(textwrap.dedent("""
rm -rf %(moodledata_path)s
# Delete tables with prefix %(site_name)s
# Delete tables with prefix %(db_prefix)s
mysql -Nrs \\
--host="%(db_host)s" \\
--user="%(db_user)s" \\
@ -114,7 +115,7 @@ class MoodleMuBackend(ServiceController):
SET @tbls = (SELECT GROUP_CONCAT(TABLE_NAME)
FROM information_schema.TABLES
WHERE TABLE_SCHEMA = "%(db_name)s"
AND TABLE_NAME LIKE "%(site_name)s_%%");
AND TABLE_NAME LIKE "%(db_prefix)s_%%");
SET @delStmt = CONCAT("DROP TABLE ", @tbls);
-- SELECT @delStmt;
PREPARE stmt FROM @delStmt;
@ -146,9 +147,10 @@ class MoodleMuBackend(ServiceController):
'db_pass': settings.SAAS_MOODLE_DB_PASS,
'db_name': settings.SAAS_MOODLE_DB_NAME,
'db_host': settings.SAAS_MOODLE_DB_HOST,
'db_prefix': saas.name.replace('-', '_'),
'email': saas.account.email,
'password': getattr(saas, 'password', None),
'custom_url': saas.custom_url,
'custom_url': saas.custom_url.rstrip('/'),
'custom_domain': urlparse(saas.custom_url).netloc if saas.custom_url else None,
}
context.update({

4
orchestra/contrib/saas/models.py
View file

@ -24,8 +24,8 @@ class SaaS(models.Model):
service = models.CharField(_("service"), max_length=32,
choices=SoftwareService.get_choices())
name = models.CharField(_("Name"), max_length=64,
help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."),
validators=[validators.validate_username])
help_text=_("Required. 64 characters or fewer. Letters, digits and ./- only."),
validators=[validators.validate_hostname])
account = models.ForeignKey('accounts.Account', verbose_name=_("account"),
related_name='saas')
is_active = models.BooleanField(_("active"), default=True,

6
orchestra/contrib/saas/services/options.py
View file

@ -1,3 +1,4 @@
from functools import lru_cache
from urllib.parse import urlparse
from django.core.exceptions import ValidationError, ObjectDoesNotExist
@ -8,7 +9,6 @@ from orchestra.contrib.databases.models import Database, DatabaseUser
from orchestra.contrib.orchestration import Operation
from orchestra.contrib.websites.models import Website, WebsiteDirective
from orchestra.utils.apps import isinstalled
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class
from . import helpers
@ -33,7 +33,7 @@ class SoftwareService(plugins.Plugin):
allow_custom_url = False
@classmethod
@cached
@lru_cache()
def get_plugins(cls):
plugins = []
for cls in settings.SAAS_ENABLED_SERVICES:
@ -153,7 +153,7 @@ class DBSoftwareService(SoftwareService):
def get_db_user(self):
return self.db_user
@cached
@lru_cache()
def get_account(self):
account_model = self.instance._meta.get_field_by_name('account')[0]
return account_model.rel.to.objects.get_main()

3
orchestra/contrib/services/handlers.py
View file

@ -2,6 +2,7 @@ import calendar
import datetime
import decimal
import math
from functools import cmp_to_key
from dateutil import relativedelta
from django.contrib.contenttypes.models import ContentType
@ -11,7 +12,7 @@ from django.utils.translation import ugettext, ugettext_lazy as _
from orchestra import plugins
from orchestra.utils.humanize import text2int
from orchestra.utils.python import AttrDict, cmp_to_key, format_exception
from orchestra.utils.python import AttrDict, format_exception
from . import settings, helpers

2
orchestra/contrib/services/tests/test_handler.py
View file

@ -1,12 +1,12 @@
import datetime
import decimal
from functools import cmp_to_key
from django.contrib.contenttypes.models import ContentType
from django.utils import timezone
from orchestra.contrib.systemusers.models import SystemUser
from orchestra.contrib.plans.models import Plan
from orchestra.utils.python import cmp_to_key
from orchestra.utils.tests import BaseTestCase
from .. import helpers

1
orchestra/contrib/systemusers/backends.py
View file

@ -31,6 +31,7 @@ class UNIXUserBackend(ServiceController):
groups = ','.join(self.get_groups(user))
context['groups_arg'] = '--groups %s' % groups if groups else ''
# TODO userd add will fail if %(user)s group already exists
# TODO mkhomedir_helper
self.append(textwrap.dedent("""
# Update/create user state for %(user)s
if id %(user)s ; then

1
orchestra/contrib/webapps/models.py
View file

@ -7,7 +7,6 @@ from django.utils.translation import ugettext_lazy as _
from jsonfield import JSONField
from orchestra.core import validators
from orchestra.utils.functional import cached
from . import settings
from .fields import VirtualDatabaseRelation, VirtualDatabaseUserRelation

6
orchestra/contrib/webapps/options.py
View file

@ -1,10 +1,10 @@
import re
from functools import lru_cache
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from orchestra.plugins import Plugin
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class
from . import settings
@ -20,7 +20,7 @@ class AppOption(Plugin):
comma_separated = False
@classmethod
@cached
@lru_cache()
def get_plugins(cls):
plugins = []
for cls in settings.WEBAPPS_ENABLED_OPTIONS:
@ -28,7 +28,7 @@ class AppOption(Plugin):
return plugins
@classmethod
@cached
@lru_cache()
def get_option_groups(cls):
groups = {}
for opt in cls.get_plugins():

9
orchestra/contrib/webapps/types/__init__.py
View file

@ -1,9 +1,10 @@
from functools import lru_cache
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from orchestra import plugins
from orchestra.plugins.forms import PluginDataForm
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class
from .. import settings
@ -22,7 +23,7 @@ class AppType(plugins.Plugin):
# TODO generic name like 'execution' ?
@classmethod
@cached
@lru_cache()
def get_plugins(cls):
plugins = []
for cls in settings.WEBAPPS_TYPES:
@ -38,7 +39,7 @@ class AppType(plugins.Plugin):
})
@classmethod
@cached
@lru_cache()
def get_group_options(cls):
""" Get enabled options based on cls.option_groups """
groups = AppOption.get_option_groups()
@ -54,7 +55,7 @@ class AppType(plugins.Plugin):
@classmethod
def get_group_options_choices(cls):
""" Generates grouped choices ready to use in Field.choices """
# generators can not be @cached
# generators can not be @lru_cache
yield (None, '-------')
for group, options in cls.get_group_options():
if group is None:

4
orchestra/contrib/webapps/types/php.py
View file

@ -1,12 +1,12 @@
import os
from collections import OrderedDict
from functools import lru_cache
from django import forms
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from orchestra.plugins.forms import PluginDataForm
from orchestra.utils.functional import cached
from .. import settings, utils
from ..options import AppOption
@ -58,7 +58,7 @@ class PHPApp(AppType):
def get_detail(self):
return self.instance.data.get('php_version', '')
@cached
@lru_cache()
def get_options(self, merge=settings.WEBAPPS_MERGE_PHP_WEBAPPS):
""" adapter to webapp.get_options that performs merging of PHP options """
kwargs = {

8
orchestra/contrib/websites/directives.py
View file

@ -1,11 +1,11 @@
import re
from collections import defaultdict
from functools import lru_cache
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from orchestra.plugins import Plugin
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class
from . import settings
@ -24,7 +24,7 @@ class SiteDirective(Plugin):
unique_location = False
@classmethod
@cached
@lru_cache()
def get_plugins(cls):
plugins = []
for cls in settings.WEBSITES_ENABLED_DIRECTIVES:
@ -32,7 +32,7 @@ class SiteDirective(Plugin):
return plugins
@classmethod
@cached
@lru_cache()
def get_option_groups(cls):
groups = {}
for opt in cls.get_plugins():
@ -45,7 +45,7 @@ class SiteDirective(Plugin):
@classmethod
def get_choices(cls):
""" Generates grouped choices ready to use in Field.choices """
# generators can not be @cached
# generators can not be @lru_cache()
yield (None, '-------')
options = cls.get_option_groups()
for option in options.pop(None, ()):

4
orchestra/contrib/websites/models.py
View file

@ -1,12 +1,12 @@
import os
from collections import OrderedDict
from functools import lru_cache
from django.db import models
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from orchestra.core import validators
from orchestra.utils.functional import cached
from . import settings
from .directives import SiteDirective
@ -72,7 +72,7 @@ class Website(models.Model):
return self.HTTP
return self.HTTPS
@cached
@lru_cache()
def get_directives(self):
directives = OrderedDict()
for opt in self.directives.all().order_by('name', 'value'):

18
orchestra/core/validators.py
View file

@ -1,11 +1,11 @@
import logging
import re
from ipaddress import ip_address
import phonenumbers
from django.core import validators
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from IPy import IP
from ..utils.python import import_class
@ -40,28 +40,28 @@ def all_valid(*args):
def validate_ipv4_address(value):
msg = _("Not a valid IPv4 address")
try:
ip = IP(value)
except:
ip = ip_address(value)
except ValueError:
raise ValidationError(msg)
if ip.version() != 4:
if ip.version != 4:
raise ValidationError(msg)
def validate_ipv6_address(value):
msg = _("Not a valid IPv6 address")
try:
ip = IP(value)
except:
ip = ip_address(value)
except ValueError:
raise ValidationError(msg)
if ip.version() != 6:
if ip.version != 6:
raise ValidationError(msg)
def validate_ip_address(value):
msg = _("Not a valid IP address")
try:
IP(value)
except:
ip_address(value)
except ValueError:
raise ValidationError(msg)

38
orchestra/management/commands/setupcelery.py
View file

@ -4,7 +4,8 @@ from os import path
from django.core.management.base import BaseCommand
from orchestra.utils.paths import get_site_dir, get_orchestra_dir
from orchestra.contrib.settings import Setting, parser as settings_parser
from orchestra.utils.paths import get_site_dir, get_orchestra_dir, get_project_dir
from orchestra.utils.sys import run, check_root
@ -32,6 +33,7 @@ class Command(BaseCommand):
'username': options.get('username'),
'bin_path': path.join(get_orchestra_dir(), 'bin'),
'processes': options.get('processes'),
'settings': path.join(get_project_dir(), 'settings.py')
}
celery_config = textwrap.dedent("""\
@ -103,3 +105,37 @@ class Command(BaseCommand):
}"""
)
run("echo '%s' > /etc/logrotate.d/celeryd" % rotate)
changes = {}
if Setting.settings['TASKS_BACKEND'].value != 'celery':
changes['TASKS_BACKEND'] = 'celery'
if Setting.settings['ORCHESTRA_START_SERVICES'].value == Setting.settings['ORCHESTRA_START_SERVICES'].default:
changes['ORCHESTRA_START_SERVICES'] = settings_parser.serialize(
(
'postgresql',
'celeryevcam',
'celeryd',
'celerybeat',
('uwsgi', 'nginx'),
)
)
if Setting.settings['ORCHESTRA_RESTART_SERVICES'].value == Setting.settings['ORCHESTRA_RESTART_SERVICES'].default:
changes['ORCHESTRA_RESTART_SERVICES'] = settings_parser.serialize(
(
'celeryd',
'celerybeat',
'uwsgi',
)
)
if Setting.settings['ORCHESTRA_STOP_SERVICES'].value == Setting.settings['ORCHESTRA_STOP_SERVICES'].default:
changes['ORCHESTRA_STOP_SERVICES'] = settings_parser.serialize(
(
('uwsgi', 'nginx'),
'celerybeat',
'celeryd',
'celeryevcam',
'postgresql'
)
)
if changes:
settings_parser.apply(changes)

14
orchestra/management/commands/setupcronbeat.py
View file

@ -2,6 +2,7 @@ import os
from django.core.management.base import BaseCommand
from orchestra.contrib.settings import Setting, parser as settings_parser
from orchestra.utils.paths import get_site_dir
from orchestra.utils.sys import run, check_non_root
@ -24,3 +25,16 @@ class Command(BaseCommand):
content += "\n* * * * * %(orchestra_beat)s %(site_dir)s/manage.py" % context
context['content'] = content
run("cat << EOF | crontab\n%(content)s\nEOF" % context, display=True)
# Configrue settings to use threaded task backend
changes = {}
if Setting.settings['TASKS_BACKEND'].value == 'celery':
changes['TASKS_BACKEND'] = settings_parser.Remove()
if 'celeryd' in Setting.settings['ORCHESTRA_START_SERVICES'].value:
changes['ORCHESTRA_START_SERVICES'] = settings_parser.Remove()
if 'celeryd' in Setting.settings['ORCHESTRA_RESTART_SERVICES'].value:
changes['ORCHESTRA_RESTART_SERVICES'] = settings_parser.Remove()
if 'celeryd' in Setting.settings['ORCHESTRA_STOP_SERVICES'].value:
changes['ORCHESTRA_STOP_SERVICES'] = settings_parser.Remove()
if changes:
settings_parser.apply(changes)

15
orchestra/management/commands/setuppostgres.py
View file

@ -48,14 +48,12 @@ class Command(BaseCommand):
help = 'Setup PostgreSQL database.'
def run_postgres(self, cmd, *args, **kwargs):
return run('su postgres -c "psql -c \\"%s\\""' % cmd, *args, display=True, **kwargs)
return run('su postgres -c "psql -c \\"%s\\""' % cmd, *args, **kwargs)
@check_root
def handle(self, *args, **options):
interactive = options.get('interactive')
db_password = options.get('db_password')
print(db_password)
print(type(db_password))
context = {
'db_name': options.get('db_name'),
'db_user': options.get('db_user'),
@ -76,11 +74,11 @@ class Command(BaseCommand):
"please provide a password [%(default_db_password)s]: " % context)
context['db_password'] = input(msg) or context['default_db_password']
self.run_postgres(alter_user % context)
msg = "Updated Postgres user '%(db_user)s' password '%(db_password)s'"
msg = "Updated Postgres user '%(db_user)s' password: '%(db_password)s'"
self.stdout.write(msg % context)
elif db_password:
self.run_postgres(alter_user % context)
msg = "Updated Postgres user '%(db_user)s' password '%(db_password)s'"
msg = "Updated Postgres user '%(db_user)s' password: '%(db_password)s'"
self.stdout.write(msg % context)
else:
raise CommandError("Postgres user '%(db_user)s' already exists and "
@ -90,13 +88,6 @@ class Command(BaseCommand):
self.stdout.write(msg % context)
self.run_postgres(create_database % context, valid_codes=(0,1))
# run(textwrap.dedent("""\
# su postgres -c "psql -c \\"CREATE USER %(db_user)s PASSWORD '%(db_password)s';\\"" || {
# su postgres -c "psql -c \\"ALTER USER %(db_user)s WITH PASSWORD '%(db_password)s';\\""
# }
# su postgres -c "psql -c \\"CREATE DATABASE %(db_name)s OWNER %(db_user)s;\\""\
# """) % context, valid_codes=(0,1)
# )
context.update({
'settings': os.path.join(get_project_dir(), 'settings.py')
})

5
orchestra/utils/functional.py
View file

@ -1,5 +1,8 @@
def cached(func):
""" caches func return value """
"""
DEPRECATED in favour of lru_cahce
caches func return value
"""
def cached_func(self, *args, **kwargs):
# id(self) prevents sharing within subclasses
attr = '_cached_%s_%i' % (func.__name__, id(self))

20
orchestra/utils/python.py
View file

@ -100,26 +100,6 @@ class CaptureStdout(list):
sys.stdout = self._stdout
def cmp_to_key(mycmp):
'Convert a cmp= function into a key= function'
class K(object):
def __init__(self, obj, *args):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj) < 0
def __gt__(self, other):
return mycmp(self.obj, other.obj) > 0
def __eq__(self, other):
return mycmp(self.obj, other.obj) == 0
def __le__(self, other):
return mycmp(self.obj, other.obj) <= 0
def __ge__(self, other):
return mycmp(self.obj, other.obj) >= 0
def __ne__(self, other):
return mycmp(self.obj, other.obj) != 0
return K
def pairwise(iterable):
"s -> (s0,s1), (s1,s2), (s2, s3), ..."
a, b = tee(iterable)

1
requirements.txt
View file

@ -2,7 +2,6 @@ django==1.8.2
django-celery-email==1.0.4
django-fluent-dashboard==0.5
https://github.com/glic3rinu/django-admin-tools/archive/master.zip
IPy==0.81
django-extensions==1.5.2
django-transaction-signals==1.0.0
django-celery==3.1.16

6
scripts/containers/deploy.sh
View file

@ -40,11 +40,11 @@ function main () {
# TODO setupceleryd shoudl change orchestra_start/stop/restart_services
while true; do
read -p "Do you want to use celery or cronbeat for task execution [cronbeat]? " task
read -p "Do you want to use celery or cronbeat (orchestra.contrib.tasks) for task execution [cronbeat]? " task
case $task in
'celery' ) task=celery; break;;
'cronbeat' ) task=cronbeat; break;;
'' ) task=cronbeat; break;;
'orchestra.contrib.tasks' ) task=orchestra.contrib.tasks; break;;
'' ) task=orchestra.contrib.tasks; break;;
* ) echo "Please answer celery or cronbeat.";;
esac
done