pedro
/
django-orchestra-test
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ported to python 3.4

This commit is contained in:
Marc Aymerich 2015-04-02 16:14:55 +00:00
parent b29c554878
commit b3946168f3
112 changed files with 413 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
TODO.md
View File

@ -236,7 +236,6 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl
* display subline links on billlines, to show that they exists.
* update service orders on a celery task? because it take alot
*
* billline quantity eval('10x100') instead of miningless description '(10*100)'
* IMPORTANT do more test, make sure billed until doesn't get uodated whhen services are billed with les metric, and don't upgrade billed_until when undoing under this circumstances
@ -245,8 +244,6 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl
* threshold for significative metric accountancy on services.handler
* http://orchestra.pangea.org/admin/orders/order/6418/
* http://orchestra.pangea.org/admin/orders/order/6495/bill_selected_orders/
* >>> round(float(decimal.Decimal('2.63'))/0.5)*0.5
* >>> round(float(str(decimal.Decimal('2.99')).split('.')[0]))/1*1
* move normurlpath to orchestra.utils from websites.utils
@ -286,23 +283,13 @@ translation.activate('ca')
ugettext("Description")
Object = disk*15
bscw quota
root@web:/home/pangea/bscw/bin ./bsadmin quota report
Disk Objects
User usage soft hard time usage soft hard time
xxx2 -- 0 20M 22M 9 200 300
xxxxxxxxxxxxx -- 0 20M 22M 8 200 300
xxxxx -- 0 20M 22M 7 200 300
xxxxx -- 0 20M 22M 7 200 300
* saas validate_creation generic approach, for all backends. standard output
* html code x: ×
* cleanup backendlogs, monitor data and metricstorage
* periodic task to cleanup backendlogs, monitor data and metricstorage
* create orchestrate databases.Database pk=1 -n --dry-run | --noinput --action save (default)|delete --backend name (limit to this backend) --help
* uwsgi --max-requests=5000 \ # respawn processes after serving 5000 requests and
@ -313,3 +300,19 @@ celery max-tasks-per-child
* postupgradeorchestra send signals in order to hook custom stuff
* make base home for systemusers that ara homed into main account systemuser
* user force_text instead of unicode for _()
* autoscale celery workers http://docs.celeryproject.org/en/latest/userguide/workers.html#autoscaling
* Delete transaction middleware
* webapp has_website list filter
apt-get install python3 python3-pip
cp /usr/local/lib/python2.7/dist-packages/orchestra.pth /usr/local/lib/python3.4/dist-packages/
glic3rinu's django-fluent-dashboard

2
docs/conf.py
View File

@ -1,3 +1,5 @@
from __future__ import unicode_literals
# -*- coding: utf-8 -*-
#
# django-orchestra documentation build configuration file, created by

4
orchestra/admin/__init__.py
View File

@ -1,2 +1,2 @@
from options import *
from dashboard import *
from .options import *
from .dashboard import *

2
orchestra/admin/actions.py
View File

@ -100,7 +100,7 @@ class SendEmail(object):
'content_message': _(
"Are you sure you want to send the following message to the following %s?"
) % self.opts.verbose_name_plural,
'display_objects': [u"%s (%s)" % (contact, contact.email) for contact in self.queryset],
'display_objects': ["%s (%s)" % (contact, contact.email) for contact in self.queryset],
'form': form,
'subject': subject,
'message': message,

2
orchestra/admin/dashboard.py
View File

@ -5,7 +5,7 @@ from orchestra.core import services
def generate_services_group():
models = []
for model, options in services.get().iteritems():
for model, options in services.get().items():
if options.get('menu', True):
models.append("%s.%s" % (model.__module__, model._meta.object_name))

8
orchestra/admin/forms.py
View File

@ -17,7 +17,7 @@ class AdminFormMixin(object):
def as_admin(self):
prepopulated_fields = {}
fieldsets = [
(None, {'fields': self.fields.keys()})
(None, {'fields': list(self.fields.keys())})
]
adminform = helpers.AdminForm(self, fieldsets, prepopulated_fields)
template = Template(
@ -32,7 +32,7 @@ class AdminFormSet(BaseModelFormSet):
def as_admin(self):
prepopulated = {}
fieldsets = [
(None, {'fields': self.form().fields.keys()})
(None, {'fields': list(self.form().fields.keys())})
]
readonly = getattr(self.form.Meta, 'readonly_fields', ())
if not hasattr(self.modeladmin, 'verbose_name_plural'):
@ -114,7 +114,11 @@ class AdminPasswordChangeForm(forms.Form):
if password:
self.user.set_password(password)
if commit:
try:
self.user.save(update_fields=['password'])
except ValueError:
# password is not a field but an attribute
self.user.save() # Trigger the backend
for ix, rel in enumerate(self.related):
password = self.cleaned_data['password1_%s' % ix]
if password:

4
orchestra/admin/menu.py
View File

@ -32,7 +32,7 @@ def api_link(context):
def get_services():
childrens = []
for model, options in services.get().iteritems():
for model, options in services.get().items():
if options.get('menu', True):
opts = model._meta
url = reverse('admin:{}_{}_changelist'.format(
@ -50,7 +50,7 @@ def get_accounts():
if isinstalled('orchestra.apps.issues'):
url = reverse('admin:issues_ticket_changelist')
childrens.append(items.MenuItem(_("Tickets"), url))
for model, options in accounts.get().iteritems():
for model, options in accounts.get().items():
if options.get('menu', True):
opts = model._meta
url = reverse('admin:{}_{}_changelist'.format(

2
orchestra/admin/options.py
View File

@ -80,7 +80,7 @@ class ChangeViewActionsMixin(object):
""" allow customization on modelamdin """
views = []
for action in self.change_view_actions:
if isinstance(action, basestring):
if isinstance(action, str):
action = getattr(self, action)
view = action_to_view(action, self)
view.url_name = getattr(action, 'url_name', action.__name__)

4
orchestra/admin/utils.py
View File

@ -20,7 +20,7 @@ from .decorators import admin_field
def get_modeladmin(model, import_module=True):
""" returns the modeladmin registred for model """
for k,v in admin.site._registry.iteritems():
for k,v in admin.site._registry.items():
if k is model:
return v
if import_module:
@ -97,7 +97,7 @@ def change_url(obj):
@admin_field
def admin_link(*args, **kwargs):
instance = args[-1]
if kwargs['field'] in ['id', 'pk', '__unicode__']:
if kwargs['field'] in ['id', 'pk', '__str__']:
obj = instance
else:
try:

4
orchestra/api/__init__.py
View File

@ -1,2 +1,2 @@
from options import *
from actions import *
from .options import *
from .actions import *

2
orchestra/api/actions.py
View File

@ -10,7 +10,7 @@ class SetPasswordApiMixin(object):
def set_password(self, request, pk):
obj = self.get_object()
data = request.DATA
if isinstance(data, basestring):
if isinstance(data, str):
data = {
'password': data
}

6
orchestra/api/fields.py
View File

@ -20,17 +20,17 @@ class OptionField(serializers.WritableField):
properties = serializers.RelationsList()
if value:
model = getattr(parent.opts.model, self.source or 'options').related.model
if isinstance(value, basestring):
if isinstance(value, str):
try:
value = json.loads(value)
except:
raise exceptions.ParseError("Malformed property: %s" % str(value))
if not related_manager:
# POST (new parent object)
return [ model(name=n, value=v) for n,v in value.iteritems() ]
return [ model(name=n, value=v) for n,v in value.items() ]
# PUT
to_save = []
for (name, value) in value.iteritems():
for (name, value) in value.items():
try:
# Update existing property
prop = related_manager.get(name=name)

2
orchestra/api/serializers.py
View File

@ -24,7 +24,7 @@ class HyperlinkedModelSerializer(serializers.HyperlinkedModelSerializer):
""" removes postonly_fields from attrs when not posting """
model_attrs = dict(**attrs)
if instance is not None:
for attr, value in attrs.iteritems():
for attr, value in attrs.items():
if attr in self.opts.postonly_fields:
model_attrs.pop(attr)
return super(HyperlinkedModelSerializer, self).restore_object(model_attrs, instance)

6
orchestra/apps/accounts/actions.py
View File

@ -44,7 +44,7 @@ def service_report(modeladmin, request, queryset):
accounts = []
fields = []
# First we get related manager names to fire a prefetch related
for name, field in queryset.model._meta._name_map.iteritems():
for name, field in queryset.model._meta._name_map.items():
model = field[0].model
if model in services.get() and model != queryset.model:
fields.append((model, name))
@ -63,3 +63,7 @@ def service_report(modeladmin, request, queryset):
'date': timezone.now().today()
}
return render(request, settings.ACCOUNTS_SERVICE_REPORT_TEMPLATE, context)
def delete_related_services(modeladmin, request, queryset):
pass

4
orchestra/apps/accounts/forms.py
View File

@ -62,12 +62,12 @@ def create_account_creation_form():
field_name = 'create_%s' % model._meta.model_name
if self.cleaned_data[field_name]:
kwargs = {
key: eval(value, {'account': account}) for key, value in related_kwargs.iteritems()
key: eval(value, {'account': account}) for key, value in related_kwargs.items()
}
model.objects.create(account=account, **kwargs)
fields.update({
'create_related_fields': fields.keys(),
'create_related_fields': list(fields.keys()),
'clean': clean,
'save_model': save_model,
'save_related': save_related,

2
orchestra/apps/accounts/models.py
View File

@ -44,7 +44,7 @@ class Account(auth.AbstractBaseUser):
USERNAME_FIELD = 'username'
REQUIRED_FIELDS = ['email']
def __unicode__(self):
def __str__(self):
return self.name
@property

1
orchestra/apps/accounts/settings.py
View File

@ -11,6 +11,7 @@ ACCOUNTS_TYPES = getattr(settings, 'ACCOUNTS_TYPES', (
('COMPANY', _("Company")),
('PUBLICBODY', _("Public body")),
('STAFF', _("Staff")),
('FRIEND', _("Friend")),
))

8
orchestra/apps/bills/actions.py
View File

@ -1,5 +1,5 @@
import StringIO
import zipfile
from io import StringIO
from django.contrib import messages
from django.contrib.admin import helpers
@ -20,7 +20,7 @@ from .helpers import validate_contact
def download_bills(modeladmin, request, queryset):
if queryset.count() > 1:
stringio = StringIO.StringIO()
stringio = StringIO()
archive = zipfile.ZipFile(stringio, 'w')
for bill in queryset:
pdf = html_to_pdf(bill.html or bill.render())
@ -122,7 +122,7 @@ def undo_billing(modeladmin, request, queryset):
except KeyError:
group[line.order] = [line]
# TODO force incomplete info
for order, lines in group.iteritems():
for order, lines in group.items():
# Find path from ini to end
for attr in ['order_id', 'order_billed_on', 'order_billed_until']:
if not getattr(self, attr):
@ -131,7 +131,7 @@ def undo_billing(modeladmin, request, queryset):
if 'a' != order.billed_on:
raise ValidationError(_("Dates don't match"))
prev = order.billed_on
for ix in xrange(0, len(lines)):
for ix in range(0, len(lines)):
if lines[ix].order_b: # TODO we need to look at the periods here
pass
order.billed_until = self.order_billed_until

4
orchestra/apps/bills/admin.py
View File

@ -210,8 +210,8 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin):
def get_inline_instances(self, request, obj=None):
inlines = super(BillAdmin, self).get_inline_instances(request, obj)
if obj and not obj.is_open:
return [inline for inline in inlines if not type(inline) == BillLineInline]
return [inline for inline in inlines if not type(inline) == ClosedBillLineInline]
return [inline for inline in inlines if not isinstance(inline, BillLineInline)]
return [inline for inline in inlines if not isinstance(inline, ClosedBillLineInline)]
def formfield_for_dbfield(self, db_field, **kwargs):
""" Make value input widget bigger """

2
orchestra/apps/bills/forms.py
View File

@ -33,7 +33,7 @@ class SelectSourceForm(forms.ModelForm):
choices.append((source.pk, str(source)))
self.fields['source'].choices = choices
self.fields['source'].initial = choices[-1][0]
self.fields['bill_link'].initial = admin_link('__unicode__')(bill)
self.fields['bill_link'].initial = admin_link('__str__')(bill)
self.fields['display_type'].initial = bill.get_type_display()
def clean_source(self):

8
orchestra/apps/bills/models.py
View File

@ -31,7 +31,7 @@ class BillContact(models.Model):
default=settings.BILLS_CONTACT_DEFAULT_COUNTRY)
vat = models.CharField(_("VAT number"), max_length=64)
def __unicode__(self):
def __str__(self):
return self.name
def get_name(self):
@ -98,7 +98,7 @@ class Bill(models.Model):
class Meta:
get_latest_by = 'id'
def __unicode__(self):
def __str__(self):
return self.number
@cached_property
@ -235,7 +235,7 @@ class Bill(models.Model):
def get_total(self):
total = 0
for tax, subtotal in self.get_subtotals().iteritems():
for tax, subtotal in self.get_subtotals().items():
subtotal, taxes = subtotal
total += subtotal + taxes
return total
@ -287,7 +287,7 @@ class BillLine(models.Model):
amended_line = models.ForeignKey('self', verbose_name=_("amended line"),
related_name='amendment_lines', null=True, blank=True)
def __unicode__(self):
def __str__(self):
return "#%i" % self.number
@cached_property

2
orchestra/apps/bills/settings.py
View File

@ -90,7 +90,7 @@ BILLS_CONTACT_DEFAULT_CITY = getattr(settings, 'BILLS_CONTACT_DEFAULT_CITY',
BILLS_CONTACT_COUNTRIES = getattr(settings, 'BILLS_CONTACT_COUNTRIES',
((k,v) for k,v in data.COUNTRIES.iteritems())
((k,v) for k,v in data.COUNTRIES.items())
)

2
orchestra/apps/contacts/admin.py
View File

@ -62,7 +62,7 @@ class ContactAdmin(AccountAdminMixin, ExtendedModelAdmin):
actions = [SendEmail(),]
def dispaly_name(self, contact):
return unicode(contact)
return str(contact)
dispaly_name.short_description = _("Name")
dispaly_name.admin_order_field = 'short_name'

2
orchestra/apps/contacts/models.py
View File

@ -55,7 +55,7 @@ class Contact(models.Model):
choices=settings.CONTACTS_COUNTRIES,
default=settings.CONTACTS_DEFAULT_COUNTRY)
def __unicode__(self):
def __str__(self):
return self.full_name or self.short_name
def clean(self):

2
orchestra/apps/contacts/settings.py
View File

@ -18,7 +18,7 @@ CONTACTS_DEFAULT_CITY = getattr(settings, 'CONTACTS_DEFAULT_CITY',
CONTACTS_COUNTRIES = getattr(settings, 'CONTACTS_COUNTRIES', (
(k,v) for k,v in data.COUNTRIES.iteritems()
(k,v) for k,v in data.COUNTRIES.items()
))

6
orchestra/apps/databases/models.py
View File

@ -26,7 +26,7 @@ class Database(models.Model):
class Meta:
unique_together = ('name', 'type')
def __unicode__(self):
def __str__(self):
return "%s" % self.name
@property
@ -59,7 +59,7 @@ class DatabaseUser(models.Model):
verbose_name_plural = _("DB users")
unique_together = ('username', 'type')
def __unicode__(self):
def __str__(self):
return self.username
def get_username(self):
@ -68,7 +68,7 @@ class DatabaseUser(models.Model):
def set_password(self, password):
if self.type == self.MYSQL:
# MySQL stores sha1(sha1(password).binary).hex
binary = hashlib.sha1(password).digest()
binary = hashlib.sha1(password.encode('utf-8')).digest()
hexdigest = hashlib.sha1(binary).hexdigest()
self.password = '*%s' % hexdigest.upper()
else:

2
orchestra/apps/domains/admin.py
View File

@ -41,7 +41,7 @@ class DomainInline(admin.TabularInline):
extra = 0
verbose_name_plural = _("Subdomains")
domain_link = admin_link('__unicode__')
domain_link = admin_link('__str__')
domain_link.short_description = _("Name")
account_link = admin_link('account')

4
orchestra/apps/domains/models.py
View File

@ -23,7 +23,7 @@ class Domain(models.Model):
serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial,
help_text=_("Serial number"))
def __unicode__(self):
def __str__(self):
return self.name
@classmethod
@ -228,7 +228,7 @@ class Record(models.Model):
type = models.CharField(_("type"), max_length=32, choices=TYPE_CHOICES)
value = models.CharField(_("value"), max_length=256)
def __unicode__(self):
def __str__(self):
return "%s %s IN %s %s" % (self.domain, self.get_ttl(), self.type, self.value)
def clean(self):

6
orchestra/apps/issues/admin.py
View File

@ -1,5 +1,3 @@
from __future__ import absolute_import
from django import forms
from django.conf.urls import patterns
from django.contrib import admin
@ -267,8 +265,8 @@ class TicketAdmin(ChangeListDefaultFilter, ExtendedModelAdmin):
changes = get_ticket_changes(self, request, ticket)
if changes:
content = markdown_formated_changes(changes)
content += request.POST[u'messages-2-0-content']
request.POST[u'messages-2-0-content'] = content
content += request.POST['messages-2-0-content']
request.POST['messages-2-0-content'] = content
ticket.mark_as_read_by(request.user)
context = {'title': "Issue #%i - %s" % (ticket.id, ticket.subject)}
context.update(extra_context or {})

4
orchestra/apps/issues/filters.py
View File

@ -22,7 +22,7 @@ class MyTicketsListFilter(SimpleListFilter):
def choices(self, cl):
""" Remove default All """
choices = iter(super(MyTicketsListFilter, self).choices(cl))
choices.next()
next(choices)
return choices
@ -52,6 +52,6 @@ class TicketStateListFilter(SimpleListFilter):
def choices(self, cl):
""" Remove default All """
choices = iter(super(TicketStateListFilter, self).choices(cl))
choices.next()
next(choices)
return choices

2
orchestra/apps/issues/helpers.py
View File

@ -11,7 +11,7 @@ def filter_actions(modeladmin, ticket, request):
del_actions.append('take')
exclude = lambda a: not (a == action or a.url_name == action)
for action in del_actions:
actions = filter(exclude, actions)
actions = list(filter(exclude, actions))
return actions

10
orchestra/apps/issues/models.py
View File

@ -20,7 +20,7 @@ class Queue(models.Model):
default=contacts_settings.CONTACTS_DEFAULT_EMAIL_USAGES,
help_text=_("Contacts to notify by email"))
def __unicode__(self):
def __str__(self):
return self.verbose_name or self.name
def save(self, *args, **kwargs):
@ -77,8 +77,8 @@ class Ticket(models.Model):
class Meta:
ordering = ['-updated_at']
def __unicode__(self):
return unicode(self.pk)
def __str__(self):
return str(self.pk)
def get_notification_emails(self):
""" Get emails of the users related to the ticket """
@ -164,8 +164,8 @@ class Message(models.Model):
class Meta:
get_latest_by = 'id'
def __unicode__(self):
return u"#%i" % self.id
def __str__(self):
return "#%i" % self.id
def save(self, *args, **kwargs):
""" notify stakeholders of ticket update """

2
orchestra/apps/lists/backends.py
View File

@ -295,7 +295,7 @@ class MailmanTraffic(ServiceMonitor):
except IOError as e:
sys.stderr.write(e)
for list_name, opts in lists.iteritems():
for list_name, opts in lists.items():
__, object_id, size = opts
if size:
cmd = ' '.join(('list_members', list_name, '| wc -l'))

2
orchestra/apps/lists/models.py
View File

@ -30,7 +30,7 @@ class List(models.Model):
class Meta:
unique_together = ('address_name', 'address_domain')
def __unicode__(self):
def __str__(self):
return self.name
@property

2
orchestra/apps/mailboxes/admin.py
View File

@ -1,5 +1,5 @@
import copy
from urlparse import parse_qs
from urllib.parse import urlparse
from django import forms
from django.contrib import admin

2
orchestra/apps/mailboxes/backends.py
View File

@ -431,7 +431,7 @@ class PostfixTraffic(ServiceMonitor):
except IOError as e:
sys.stderr.write(e)
for username, opts in users.iteritems():
for username, opts in users.items():
size = 0
for req_id in reverse[username]:
size += targets[req_id][1] * counter.get(req_id, 0)

10
orchestra/apps/mailboxes/models.py
View File

@ -25,7 +25,7 @@ class Mailbox(models.Model):
filtering = models.CharField(max_length=16,
default=settings.MAILBOXES_MAILBOX_DEFAULT_FILTERING,
choices=[
(k, v[0]) for k,v in settings.MAILBOXES_MAILBOX_FILTERINGS.iteritems()
(k, v[0]) for k,v in settings.MAILBOXES_MAILBOX_FILTERINGS.items()
])
custom_filtering = models.TextField(_("filtering"), blank=True,
validators=[validators.validate_sieve],
@ -36,7 +36,7 @@ class Mailbox(models.Model):
class Meta:
verbose_name_plural = _("mailboxes")
def __unicode__(self):
def __str__(self):
return self.name
@cached_property
@ -62,7 +62,7 @@ class Mailbox(models.Model):
def get_filtering(self):
__, filtering = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering]
if isinstance(filtering, basestring):
if isinstance(filtering, str):
return filtering
return filtering(self)
@ -104,7 +104,7 @@ class Address(models.Model):
verbose_name_plural = _("addresses")
unique_together = ('name', 'domain')
def __unicode__(self):
def __str__(self):
return self.email
@property
@ -154,7 +154,7 @@ class Autoresponse(models.Model):
message = models.TextField(_("message"))
enabled = models.BooleanField(_("enabled"), default=False)
def __unicode__(self):
def __str__(self):
return self.address

2
orchestra/apps/miscellaneous/admin.py
View File

@ -54,7 +54,7 @@ class MiscServiceAdmin(ExtendedModelAdmin):
class MiscellaneousAdmin(AccountAdminMixin, SelectPluginAdminMixin, admin.ModelAdmin):
list_display = (
'__unicode__', 'service_link', 'amount', 'dispaly_active', 'account_link'
'__str__', 'service_link', 'amount', 'dispaly_active', 'account_link'
)
list_filter = ('service__name', 'is_active')
list_select_related = ('service', 'account')

1
orchestra/apps/miscellaneous/migrations/0001_initial.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import orchestra.core.validators

1
orchestra/apps/miscellaneous/migrations/0002_auto_20141112_1853.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import orchestra.models.fields

4
orchestra/apps/miscellaneous/models.py
View File

@ -25,7 +25,7 @@ class MiscService(models.Model):
help_text=_("Whether new instances of this service can be created "
"or not. Unselect this instead of deleting services."))
def __unicode__(self):
def __str__(self):
return self.name
def clean(self):
@ -51,7 +51,7 @@ class Miscellaneous(models.Model):
class Meta:
verbose_name_plural = _("miscellaneous")
def __unicode__(self):
def __str__(self):
return self.identifier or self.description[:32] or str(self.service)
@cached_property

13
orchestra/apps/orchestration/backends.py
View File

@ -18,7 +18,7 @@ class ServiceMount(plugins.PluginMount):
super(ServiceMount, cls).__init__(name, bases, attrs)
class ServiceBackend(plugins.Plugin):
class ServiceBackend(plugins.Plugin, metaclass=ServiceMount):
"""
Service management backend base class
@ -37,13 +37,8 @@ class ServiceBackend(plugins.Plugin):
default_route_match = 'True'
block = False # Force the backend manager to block in multiple backend executions and execute them synchronously
__metaclass__ = ServiceMount
def __unicode__(self):
return type(self).__name__
def __str__(self):
return unicode(self)
return type(self).__name__
def __init__(self):
self.head = []
@ -138,7 +133,7 @@ class ServiceBackend(plugins.Plugin):
scripts[method] += commands
except KeyError:
pass
return list(scripts.iteritems())
return list(scripts.items())
def get_banner(self):
time = timezone.now().strftime("%h %d, %Y %I:%M:%S %Z")
@ -159,7 +154,7 @@ class ServiceBackend(plugins.Plugin):
def append(self, *cmd):
# aggregate commands acording to its execution method
if isinstance(cmd[0], basestring):
if isinstance(cmd[0], str):
method = self.script_method
cmd = cmd[0]
else:

5
orchestra/apps/orchestration/helpers.py
View File

@ -7,10 +7,9 @@ from django.utils.translation import ungettext, ugettext_lazy as _
def send_report(method, args, log):
backend = method.im_class().get_name()
server = args[0]
subject = '[Orchestra] %s execution %s on %s'
subject = subject % (backend, log.state, server)
backend = method.__self__.__class__.__name__
subject = '[Orchestra] %s execution %s on %s' % (backend, log.state, server)
separator = "\n%s\n\n" % ('~ '*40,)
message = separator.join([
"[EXIT CODE] %s" % log.exit_code,

2
orchestra/apps/orchestration/management/commands/orchestrate.py
View File

@ -32,7 +32,7 @@ class Command(BaseCommand):
scripts, block = manager.generate(operations)
servers = []
# Print scripts
for key, value in scripts.iteritems():
for key, value in scripts.items():
server, __ = key
backend, operations = value
servers.append(server.name)

9
orchestra/apps/orchestration/manager.py
View File

@ -1,3 +1,4 @@
import logging
import threading
import traceback
@ -47,7 +48,7 @@ def close_connection(execute):
def wrapper(*args, **kwargs):
try:
log = execute(*args, **kwargs)
except:
except Exception as e:
pass
else:
wrapper.log = log
@ -86,7 +87,7 @@ def generate(operations):
post_action.send(**kwargs)
if backend.block:
block = True
for value in scripts.itervalues():
for value in scripts.values():
backend, operations = value
backend.commit()
return scripts, block
@ -97,13 +98,13 @@ def execute(scripts, block=False, async=False):
# Execute scripts on each server
threads = []
executions = []
for key, value in scripts.iteritems():
for key, value in scripts.items():
server, __ = key
backend, operations = value
execute = as_task(backend.execute)
logger.debug('%s is going to be executed on %s' % (backend, server))
if block:
# Execute one bakend at a time, no need for threads
# Execute one backend at a time, no need for threads
execute(server, async=async)
else:
execute = close_connection(execute)

13
orchestra/apps/orchestration/methods.py
View File

@ -29,7 +29,8 @@ def SSH(backend, log, server, cmds, async=False):
"""
script = '\n'.join(cmds)
script = script.replace('\r', '')
digest = hashlib.md5(script).hexdigest()
bscript = script.encode('utf-8')
digest = hashlib.md5(bscript).hexdigest()
path = os.path.join(settings.ORCHESTRATION_TEMP_SCRIPT_PATH, digest)
remote_path = "%s.remote" % path
log.script = '# %s\n%s' % (remote_path, script)
@ -41,8 +42,8 @@ def SSH(backend, log, server, cmds, async=False):
try:
# Avoid "Argument list too long" on large scripts by genereting a file
# and scping it to the remote server
with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0600), 'w') as handle:
handle.write(script)
with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as handle:
handle.write(bscript)
# ssh connection
ssh = paramiko.SSHClient()
@ -62,7 +63,7 @@ def SSH(backend, log, server, cmds, async=False):
# Copy script to remote server
sftp = paramiko.SFTPClient.from_transport(transport)
sftp.put(path, remote_path)
sftp.chmod(remote_path, 0600)
sftp.chmod(remote_path, 0o600)
sftp.close()
os.remove(path)
@ -124,7 +125,7 @@ def SSH(backend, log, server, cmds, async=False):
def Python(backend, log, server, cmds, async=False):
# TODO collect stdout?
script = [ str(cmd.func.func_name) + str(cmd.args) for cmd in cmds ]
script = [ str(cmd.func.__name__) + str(cmd.args) for cmd in cmds ]
script = json.dumps(script, indent=4).replace('"', '')
log.script = '\n'.join([log.script, script])
log.save(update_fields=['script'])
@ -133,7 +134,7 @@ def Python(backend, log, server, cmds, async=False):
with CaptureStdout() as stdout:
result = cmd(server)
for line in stdout:
log.stdout += unicode(line, errors='replace') + '\n'
log.stdout += line + '\n'
if async:
log.save(update_fields=['stdout'])
except:

47
orchestra/apps/orchestration/middlewares.py
View File

@ -1,6 +1,7 @@
from threading import local
from django.core.urlresolvers import resolve
from django.db import connection, transaction
from django.db.models.signals import pre_delete, post_save, m2m_changed
from django.dispatch import receiver
from django.http.response import HttpResponseServerError
@ -36,6 +37,11 @@ class OperationsMiddleware(object):
"""
Stores all the operations derived from save and delete signals and executes them
at the end of the request/response cycle
It also works as a transaction middleware. Each view function will be run
with commit_on_response activated - that way a save() doesn't do a direct
commit, the commit is done when a successful response is created. If an
exception happens, the database is rolled back.
"""
# Thread local is used because request object is not available on model signals
thread_locals = local()
@ -71,16 +77,55 @@ class OperationsMiddleware(object):
instance = kwargs.pop('instance')
manager.collect(instance, action, **kwargs)
def commit_transaction(self):
if not transaction.get_autocommit():
if transaction.is_dirty():
# Note: it is possible that the commit fails. If the reason is
# closed connection or some similar reason, then there is
# little hope to proceed nicely. However, in some cases (
# deferred foreign key checks for exampl) it is still possible
# to rollback().
try:
transaction.commit()
except Exception:
# If the rollback fails, the transaction state will be
# messed up. It doesn't matter, the connection will be set
# to clean state after the request finishes. And, we can't
# clean the state here properly even if we wanted to, the
# connection is in transaction but we can't rollback...
transaction.rollback()
transaction.leave_transaction_management()
raise
transaction.leave_transaction_management()
def process_request(self, request):
""" Store request on a thread local variable """
type(self).thread_locals.request = request
# Enters transaction management
transaction.enter_transaction_management()
def process_exception(self, request, exception):
"""Rolls back the database and leaves transaction management"""
if transaction.is_dirty():
# This rollback might fail because of network failure for example.
# If rollback isn't possible it is impossible to clean the
# connection's state. So leave the connection in dirty state and
# let request_finished signal deal with cleaning the connection.
transaction.rollback()
transaction.leave_transaction_management()
def process_response(self, request, response):
""" Processes pending backend operations """
if not isinstance(response, HttpResponseServerError):
operations = type(self).get_pending_operations()
if operations:
logs = Operation.execute(operations)
scripts, block = manager.generate(operations)
# We commit transaction just before executing operations
# because here is when IntegrityError show up
self.commit_transaction()
logs = manager.execute(scripts, block=block)
if logs and resolve(request.path).app_name == 'admin':
message_user(request, logs)
return response
self.commit_transaction()
return response

13
orchestra/apps/orchestration/models.py
View File

@ -25,7 +25,7 @@ class Server(models.Model):
choices=settings.ORCHESTRATION_OS_CHOICES,
default=settings.ORCHESTRATION_DEFAULT_OS)
def __unicode__(self):
def __str__(self):
return self.name
def get_address(self):
@ -83,7 +83,7 @@ class BackendLog(models.Model):
class Meta:
get_latest_by = 'id'
def __unicode__(self):
def __str__(self):
return "%s@%s" % (self.backend, self.server)
@property
@ -116,7 +116,7 @@ class BackendOperation(models.Model):
verbose_name = _("Operation")
verbose_name_plural = _("Operations")
def __unicode__(self):
def __str__(self):
return '%s.%s(%s)' % (self.backend, self.action, self.instance)
def __hash__(self):
@ -184,7 +184,7 @@ class Route(models.Model):
class Meta:
unique_together = ('backend', 'host')
def __unicode__(self):
def __str__(self):
return "%s@%s" % (self.backend, self.host)
@property
@ -218,7 +218,7 @@ class Route(models.Model):
if not self.match:
self.match = 'True'
if self.backend:
backend_model = self.backend_class.model
backend_model = self.backend_class.model_class()
try:
obj = backend_model.objects.all()[0]
except IndexError:
@ -227,8 +227,7 @@ class Route(models.Model):
bool(self.matches(obj))
except Exception as exception:
name = type(exception).__name__
message = exception.message
raise ValidationError(': '.join((name, message)))
raise ValidationError(': '.join((name, exception)))
def matches(self, instance):
safe_locals = {

4
orchestra/apps/orders/forms.py
View File

@ -29,8 +29,8 @@ class BillSelectedOptionsForm(AdminFormMixin, forms.Form):
def selected_related_choices(queryset):
for order in queryset:
verbose = u'<a href="{order_url}">{description}</a> '
verbose += u'<a class="account" href="{account_url}">{account}</a>'
verbose = '<a href="{order_url}">{description}</a> '
verbose += '<a class="account" href="{account_url}">{account}</a>'
verbose = verbose.format(
order_url=change_url(order), description=order.description,
account_url=change_url(order.account), account=str(order.account)

36
orchestra/apps/orders/models.py
View File

@ -6,7 +6,7 @@ from django.db import models
from django.db.migrations.recorder import MigrationRecorder
from django.db.models import F, Q
from django.db.models.loading import get_model
from django.db.models.signals import post_delete, post_save
from django.db.models.signals import post_delete, post_save, pre_delete
from django.dispatch import receiver
from django.contrib.admin.models import LogEntry
from django.contrib.contenttypes import generic
@ -32,9 +32,9 @@ class OrderQuerySet(models.QuerySet):
bill_backend = Order.get_bill_backend()
qs = self.select_related('account', 'service')
commit = options.get('commit', True)
for account, services in qs.group_by('account', 'service').iteritems():
for account, services in qs.group_by('account', 'service').items():
bill_lines = []
for service, orders in services.iteritems():
for service, orders in services.items():
for order in orders:
# Saved for undoing support
order.old_billed_on = order.billed_on
@ -65,8 +65,8 @@ class OrderQuerySet(models.QuerySet):
conflictive = conflictive.exclude(service__billing_period=Service.NEVER)
conflictive = conflictive.select_related('service').group_by('account_id', 'service')
qs = Q()
for account_id, services in conflictive.iteritems():
for service, orders in services.iteritems():
for account_id, services in conflictive.items():
for service, orders in services.items():
if not service.rates.exists():
continue
ini = datetime.date.max
@ -127,8 +127,8 @@ class Order(models.Model):
class Meta:
get_latest_by = 'id'
def __unicode__(self):
return unicode(self.service)
def __str__(self):
return str(self.service)
@classmethod
def update_orders(cls, instance, service=None, commit=True):
@ -178,7 +178,7 @@ class Order(models.Model):
MetricStorage.store(self, metric)
metric = ', metric:{}'.format(metric)
description = handler.get_order_description(instance)
logger.info(u"UPDATED order id:{id}, description:{description}{metric}".format(
logger.info("UPDATED order id:{id}, description:{description}{metric}".format(
id=self.id, description=description, metric=metric).encode('ascii', 'ignore')
)
if self.description != description:
@ -247,8 +247,8 @@ class MetricStorage(models.Model):
class Meta:
get_latest_by = 'id'
def __unicode__(self):
return unicode(self.order)
def __str__(self):
return str(self.order)
@classmethod
def store(cls, order, value):
@ -268,12 +268,27 @@ class MetricStorage(models.Model):
accounts.register(Order)
@receiver(pre_delete, dispatch_uid="orders.account_orders")
def account_orders(sender, **kwargs):
account = kwargs['instance']
if isinstance(account, Order.account.field.rel.to):
account._deleted = True
# TODO build a cache hash table {model: related, model: None}
@receiver(post_delete, dispatch_uid="orders.cancel_orders")
def cancel_orders(sender, **kwargs):
if sender._meta.app_label not in settings.ORDERS_EXCLUDED_APPS:
instance = kwargs['instance']
# Account delete will delete all related orders, no need to maintain order consistency
if isinstance(instance, Order.account.field.rel.to):
# print 'aaaaaaaaaaaaaAAAAAAAAAAAAAAAAaa'
return
# print 'delete', sender, kwargs
try:
print(instance.account.pk)
except Exception as e:
pass
if type(instance) in services:
for order in Order.objects.by_object(instance).active():
order.cancel()
@ -286,6 +301,7 @@ def cancel_orders(sender, **kwargs):
def update_orders(sender, **kwargs):
if sender._meta.app_label not in settings.ORDERS_EXCLUDED_APPS:
instance = kwargs['instance']
# print 'save', sender, kwargs
if type(instance) in services:
Order.update_orders(instance)
elif not hasattr(instance, 'account'):

2
orchestra/apps/payments/actions.py
View File

@ -21,7 +21,7 @@ def process_transactions(modeladmin, request, queryset):
msg = _("Selected transactions must be on '{state}' state")
messages.error(request, msg.format(state=Transaction.WAITTING_PROCESSING))
return
for method, transactions in queryset.group_by('source__method').iteritems():
for method, transactions in queryset.group_by('source__method').items():
if method is not None:
method = PaymentMethod.get(method)
procs = method.process(transactions)

2
orchestra/apps/payments/admin.py
View File

@ -39,7 +39,7 @@ class TransactionInline(admin.TabularInline):
)
readonly_fields = fields
transaction_link = admin_link('__unicode__', short_description=_("ID"))
transaction_link = admin_link('__str__', short_description=_("ID"))
bill_link = admin_link('bill')
source_link = admin_link('source')
display_state = admin_colored('state', colors=STATE_COLORS)

2
orchestra/apps/payments/methods/sepadirectdebit.py
View File

@ -3,7 +3,7 @@ import lxml.builder
import os
from lxml import etree
from lxml.builder import E
from StringIO import StringIO
from io import StringIO
from django import forms
from django.utils import timezone

8
orchestra/apps/payments/models.py
View File

@ -26,7 +26,7 @@ class PaymentSource(models.Model):
objects = PaymentSourcesQueryset.as_manager()
def __unicode__(self):
def __str__(self):
return "%s (%s)" % (self.label, self.method_class.verbose_name)
@cached_property
@ -76,7 +76,7 @@ class TransactionQuerySet(models.QuerySet):
return self.exclude(state=Transaction.REJECTED)
def amount(self):
return self.aggregate(models.Sum('amount')).values()[0]
return next(iter(self.aggregate(models.Sum('amount')).values()))
def processing(self):
return self.filter(state__in=[Transaction.EXECUTED, Transaction.WAITTING_EXECUTION])
@ -111,7 +111,7 @@ class Transaction(models.Model):
objects = TransactionQuerySet.as_manager()
def __unicode__(self):
def __str__(self):
return "Transaction #{}".format(self.id)
@property
@ -173,7 +173,7 @@ class TransactionProcess(models.Model):
class Meta:
verbose_name_plural = _("Transaction processes")
def __unicode__(self):
def __str__(self):
return '#%i' % self.id
def check_state(*args):

8
orchestra/apps/plans/models.py
View File

@ -23,7 +23,7 @@ class Plan(models.Model):
allow_multiple = models.BooleanField(_("allow multiple"), default=False,
help_text=_("Designates whether this plan allow for multiple contractions."))
def __unicode__(self):
def __str__(self):
return self.get_verbose_name()
def clean(self):
@ -41,7 +41,7 @@ class ContractedPlan(models.Model):
class Meta:
verbose_name_plural = _("plans")
def __unicode__(self):
def __str__(self):
return str(self.plan)
def clean(self):
@ -80,7 +80,7 @@ class Rate(models.Model):
class Meta:
unique_together = ('service', 'plan', 'quantity')
def __unicode__(self):
def __str__(self):
return "{}-{}".format(str(self.price), self.quantity)
@classmethod
@ -90,7 +90,7 @@ class Rate(models.Model):
@classmethod
def get_choices(cls):
choices = []
for name, method in cls.RATE_METHODS.iteritems():
for name, method in cls.RATE_METHODS.items():
choices.append((name, method.verbose_name))
return choices

4
orchestra/apps/plans/rating.py
View File

@ -67,8 +67,8 @@ def _prepend_missing(rates):
def step_price(rates, metric):
# Step price
group = []
minimal = (sys.maxint, [])
for plan, rates in rates.group_by('plan').iteritems():
minimal = (sys.maxsize, [])
for plan, rates in rates.group_by('plan').items():
rates = _prepend_missing(rates)
value, steps = _compute(rates, metric)
if plan.is_combinable:

2
orchestra/apps/resources/actions.py
View File

@ -9,7 +9,7 @@ from django.utils.translation import ungettext, ugettext_lazy as _
def run_monitor(modeladmin, request, queryset):
""" Resource and ResourceData run monitors """
referer = request.META.get('HTTP_REFERER')
async = modeladmin.model.monitor.func_defaults[0]
async = modeladmin.model.monitor.__defaults__[0]
logs = set()
for resource in queryset:
results = resource.monitor()

2
orchestra/apps/resources/admin.py
View File

@ -262,7 +262,7 @@ def insert_resource_inlines():
if inline.__name__ == 'ResourceInline':
modeladmin_class.inlines.remove(inline)
resources = Resource.objects.filter(is_active=True)
for ct, resources in resources.group_by('content_type').iteritems():
for ct, resources in resources.group_by('content_type').items():
inline = resource_inline_factory(resources)
model = ct.model_class()
insertattr(model, 'inlines', inline)

4
orchestra/apps/resources/methods.py
View File

@ -7,10 +7,8 @@ from django.utils.translation import ugettext_lazy as _
from orchestra import plugins
class DataMethod(plugins.Plugin):
class DataMethod(plugins.Plugin, metaclass=plugins.PluginMount):
""" filters and computes dataset usage """
__metaclass__ = plugins.PluginMount
def filter(self, dataset):
""" Filter the dataset to get the relevant data according to the period """
raise NotImplementedError

1
orchestra/apps/resources/migrations/0001_initial.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import orchestra.core.validators

1
orchestra/apps/resources/migrations/0002_auto_20141117_1415.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations

8
orchestra/apps/resources/models.py
View File

@ -83,7 +83,7 @@ class Resource(models.Model):
('verbose_name', 'content_type')
)
def __unicode__(self):
def __str__(self):
return "{}-{}".format(str(self.content_type), self.name)
@cached_property
@ -188,7 +188,7 @@ class ResourceData(models.Model):
unique_together = ('resource', 'content_type', 'object_id')
verbose_name_plural = _("resource data")
def __unicode__(self):
def __str__(self):
return "%s: %s" % (str(self.resource), str(self.content_object))
@classmethod
@ -278,7 +278,7 @@ class MonitorData(models.Model):
get_latest_by = 'id'
verbose_name_plural = _("monitor data")
def __unicode__(self):
def __str__(self):
return str(self.monitor)
@cached_property
@ -331,7 +331,7 @@ def create_resource_relation():
field for field in related._meta.virtual_fields if field.rel.to != ResourceData
]
for ct, resources in Resource.objects.group_by('content_type').iteritems():
for ct, resources in Resource.objects.group_by('content_type').items():
model = ct.model_class()
relation = GenericRelation('resources.ResourceData')
model.add_to_class('resource_set', relation)

2
orchestra/apps/resources/serializers.py
View File

@ -41,7 +41,7 @@ def insert_resource_serializers():
pass
viewset.serializer_class.Meta.fields = fields
# Create nested serializers on target models
for ct, resources in Resource.objects.group_by('content_type').iteritems():
for ct, resources in Resource.objects.group_by('content_type').items():
model = ct.model_class()
try:
router.insert(model, 'resources', ResourceSerializer, required=False, many=True, source='resource_set')

4
orchestra/apps/saas/admin.py
View File

@ -2,7 +2,7 @@ from django.contrib import admin
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext, ugettext_lazy as _
from orchestra.admin import ExtendedModelAdmin
from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
from orchestra.apps.accounts.admin import AccountAdminMixin
from orchestra.plugins.admin import SelectPluginAdminMixin
@ -10,7 +10,7 @@ from .models import SaaS
from .services import SoftwareService
class SaaSAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
class SaaSAdmin(SelectPluginAdminMixin, ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
list_display = ('name', 'service', 'display_site_domain', 'account_link', 'is_active')
list_filter = ('service', 'is_active')
change_readonly_fields = ('service',)

8
orchestra/apps/saas/backends/bscw.py
View File

@ -1,3 +1,5 @@
import textwrap
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
@ -26,11 +28,11 @@ class BSCWBackend(ServiceController):
if hasattr(saas, 'password'):
self.append(textwrap.dedent("""\
if [[ ! $(%(bsadmin)s register %(email)s) && ! $(%(bsadmin)s users -n %(username)s) ]]; then
# Change password
%(bsadmin)s chpwd %(username)s '%(password)s'
else
# Create new user
%(bsadmin)s register -r %(email)s %(username)s '%(password)s'
else
# Change password
%(bsadmin)s chpwd %(username)s '%(password)s'
fi
""") % context
)

6
orchestra/apps/saas/backends/gitlab.py
View File

@ -84,7 +84,7 @@ class GitLabSaaSBackend(ServiceController):
user_url = self.get_user_url(saas)
response = requests.delete(user_url, headers=self.headers)
user = self.validate_response(response, 200, 404)
print json.dumps(user, indent=4)
print(json.dumps(user, indent=4))
def _validate_creation(self, saas, server):
""" checks if a saas object is valid for creation on the server side """
@ -95,9 +95,9 @@ class GitLabSaaSBackend(ServiceController):
users = json.loads(requests.get(users_url, headers=self.headers).content)
for user in users:
if user['username'] == username:
print 'ValidationError: user-exists'
print('ValidationError: user-exists')
if user['email'] == email:
print 'ValidationError: email-exists'
print('ValidationError: email-exists')
def validate_creation(self, saas):
self.append(self._validate_creation, saas)

2
orchestra/apps/saas/backends/phplist.py
View File

@ -34,7 +34,7 @@ class PhpListSaaSBackend(ServiceController):
'adminpassword': saas.password,
}
response = requests.post(install_link, data=post)
print response.content
print(response.content)
if response.status_code != 200:
raise RuntimeError("Bad status code %i" % response.status_code)
else:

2
orchestra/apps/saas/models.py
View File

@ -36,7 +36,7 @@ class SaaS(models.Model):
('name', 'service'),
)
def __unicode__(self):
def __str__(self):
return "%s@%s" % (self.name, self.service)
@cached_property

6
orchestra/apps/saas/services/bscw.py
View File

@ -7,18 +7,12 @@ from .. import settings
from .options import SoftwareService, SoftwareServiceForm
# TODO monitor quota since out of sync?
class BSCWForm(SoftwareServiceForm):
email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'}))
quota = forms.IntegerField(label=_("Quota"), initial=settings.SAAS_BSCW_DEFAULT_QUOTA,
help_text=_("Disk quota in MB."))
class BSCWDataSerializer(serializers.Serializer):
email = serializers.EmailField(label=_("Email"))
quota = serializers.IntegerField(label=_("Quota"), default=settings.SAAS_BSCW_DEFAULT_QUOTA,
help_text=_("Disk quota in MB."))
class BSCWService(SoftwareService):

13
orchestra/apps/saas/services/gitlab.py
View File

@ -3,7 +3,6 @@ from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from orchestra.apps.orchestration.models import BackendOperation as Operation
from orchestra.forms import widgets
from .options import SoftwareService, SoftwareServiceForm
@ -36,15 +35,3 @@ class GitLabService(SoftwareService):
verbose_name = "GitLab"
icon = 'orchestra/icons/apps/gitlab.png'
def clean_data(self):
data = super(GitLabService, self).clean_data()
if not self.instance.pk:
log = Operation.execute_action(self.instance, 'validate_creation')[0]
errors = {}
if 'user-exists' in log.stdout:
errors['name'] = _("User with this username already exists.")
elif 'email-exists' in log.stdout:
errors['email'] = _("User with this email address already exists.")
if errors:
raise ValidationError(errors)
return data

20
orchestra/apps/saas/services/options.py
View File

@ -4,9 +4,10 @@ from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from orchestra import plugins
from orchestra.plugins.forms import PluginDataForm
from orchestra.apps.orchestration.models import BackendOperation as Operation
from orchestra.core import validators
from orchestra.forms import widgets
from orchestra.plugins.forms import PluginDataForm
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class, random_ascii
@ -91,6 +92,23 @@ class SoftwareService(plugins.Plugin):
(self.instance.name, self.site_base_domain)
)
def clean_data(self):
data = super(SoftwareService, self).clean_data()
if not self.instance.pk:
try:
log = Operation.execute_action(self.instance, 'validate_creation')[0]
except IndexError:
pass
else:
errors = {}
if 'user-exists' in log.stdout:
errors['name'] = _("User with this username already exists.")
elif 'email-exists' in log.stdout:
errors['email'] = _("User with this email address already exists.")
if errors:
raise ValidationError(errors)
return data
def save(self):
pass

33
orchestra/apps/services/handlers.py
View File

@ -1,6 +1,7 @@
import calendar
import datetime
import decimal
import math
from dateutil import relativedelta
from django.contrib.contenttypes.models import ContentType
@ -15,7 +16,7 @@ from orchestra.utils.python import AttrDict
from . import settings, helpers
class ServiceHandler(plugins.Plugin):
class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
"""
Separates all the logic of billing handling from the model allowing to better
customize its behaviout
@ -27,8 +28,6 @@ class ServiceHandler(plugins.Plugin):
model = None
__metaclass__ = plugins.PluginMount
def __init__(self, service):
self.service = service
@ -54,8 +53,7 @@ class ServiceHandler(plugins.Plugin):
bool(self.matches(obj))
except Exception as exception:
name = type(exception).__name__
message = exception.message
raise ValidationError(': '.join((name, message)))
raise ValidationError(': '.join((name, exception)))
def validate_metric(self, service):
try:
@ -66,8 +64,7 @@ class ServiceHandler(plugins.Plugin):
bool(self.get_metric(obj))
except Exception as exception:
name = type(exception).__name__
message = exception.message
raise ValidationError(': '.join((name, message)))
raise ValidationError(': '.join((name, exception)))
def get_content_type(self):
if not self.model:
@ -106,18 +103,26 @@ class ServiceHandler(plugins.Plugin):
return order.ignore
def get_ignore(self, instance):
ignore = False
if self.ignore_superusers:
account = getattr(instance, 'account', instance)
if account.is_superuser:
ignore = self.ignore_superusers
return ignore
if (account.type in settings.SERVICES_IGNORE_ACCOUNT_TYPE or
'superuser' in settings.SERVICES_IGNORE_ACCOUNT_TYPE):
return True
return False
def get_metric(self, instance):
if self.metric:
safe_locals = {
instance._meta.model_name: instance
instance._meta.model_name: instance,
'instance': instance,
'math': math,
'log10': math.log10,
'Decimal': decimal.Decimal,
}
try:
return eval(self.metric, safe_locals)
except Exception as error:
raise type(error)("%s on '%s'" %(error, self.service))
def get_order_description(self, instance):
safe_locals = {
@ -126,7 +131,7 @@ class ServiceHandler(plugins.Plugin):
instance._meta.model_name: instance,
}
if not self.order_description:
return u'%s: %s' % (self.description, instance)
return '%s: %s' % (self.description, instance)
return eval(self.order_description, safe_locals)
def get_billing_point(self, order, bp=None, **options):
@ -359,7 +364,7 @@ class ServiceHandler(plugins.Plugin):
else:
priced[order] = (price, cprice)
lines = []
for order, prices in priced.iteritems():
for order, prices in priced.items():
discounts = ()
# Generate lines and discounts from order.nominal_price
price, cprice = prices

11
orchestra/apps/services/models.py
View File

@ -24,6 +24,7 @@ autodiscover_modules('handlers')
rate_class = import_class(settings.SERVICES_RATE_CLASS)
class Service(models.Model):
NEVER = ''
# DAILY = 'DAILY'
@ -46,6 +47,8 @@ class Service(models.Model):
PREPAY = 'PREPAY'
POSTPAY = 'POSTPAY'
_ignore_types = ' and '.join(', '.join(settings.SERVICES_IGNORE_ACCOUNT_TYPE).rsplit(', ', 1)).lower()
description = models.CharField(_("description"), max_length=256, unique=True)
content_type = models.ForeignKey(ContentType, verbose_name=_("content type"),
help_text=_("Content type of the related service objects."))
@ -66,8 +69,8 @@ class Service(models.Model):
"here allow to."),
choices=ServiceHandler.get_choices())
is_active = models.BooleanField(_("active"), default=True)
ignore_superusers = models.BooleanField(_("ignore superusers"), default=True,
help_text=_("Designates whether superuser orders are marked as ignored by default or not."))
ignore_superusers = models.BooleanField(_("ignore %s") % _ignore_types, default=True,
help_text=_("Designates whether %s orders are marked as ignored by default or not.") % _ignore_types)
# Billing
billing_period = models.CharField(_("billing period"), max_length=16,
help_text=_("Renewal period for recurring invoicing."),
@ -133,7 +136,7 @@ class Service(models.Model):
rate_algorithm = models.CharField(_("rate algorithm"), max_length=16,
help_text=string_concat(_("Algorithm used to interprete the rating table."), *[
string_concat('<br>&nbsp;&nbsp;', method.verbose_name, ': ', method.help_text)
for name, method in rate_class.get_methods().iteritems()
for name, method in rate_class.get_methods().items()
]), choices=rate_class.get_choices(), default=rate_class.get_choices()[0][0])
on_cancel = models.CharField(_("on cancel"), max_length=16,
help_text=_("Defines the cancellation behaviour of this service."),
@ -153,7 +156,7 @@ class Service(models.Model):
),
default=PREPAY)
def __unicode__(self):
def __str__(self):
return self.description
@classmethod

7
orchestra/apps/services/settings.py
View File

@ -33,3 +33,10 @@ SERVICES_RATE_CLASS = getattr(settings, 'SERVICES_RATE_CLASS',
SERVICES_DEFAULT_IGNORE_PERIOD = getattr(settings, 'SERVICES_DEFAULT_IGNORE_PERIOD',
'TEN_DAYS'
)
SERVICES_IGNORE_ACCOUNT_TYPE = getattr(settings, 'SERVICES_IGNORE_ACCOUNT_TYPE', (
'superuser',
'STAFF',
'FRIEND',
))

4
orchestra/apps/systemusers/backends.py
View File

@ -222,7 +222,7 @@ class Exim4Traffic(ServiceMonitor):
except IOError as e:
sys.stderr.write(e)
for username, opts in users.iteritems():
for username, opts in users.items():
__, object_id, size = opts
print object_id, size
""").format(**context)
@ -317,7 +317,7 @@ class FTPTraffic(ServiceMonitor):
except IOError as e:
sys.stderr.write(e)
for username, opts in users.iteritems():
for username, opts in users.items():
__, object_id, size = opts
print object_id, size
""").format(**context)

1
orchestra/apps/systemusers/migrations/0001_initial.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
from django.conf import settings

1
orchestra/apps/systemusers/migrations/0002_systemuser_relative_to_main.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations

1
orchestra/apps/systemusers/migrations/0003_auto_20141114_1340.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations

2
orchestra/apps/systemusers/models.py
View File

@ -48,7 +48,7 @@ class SystemUser(models.Model):
objects = SystemUserQuerySet.as_manager()
def __unicode__(self):
def __str__(self):
return self.username
@cached_property

2
orchestra/apps/vps/models.py
View File

@ -24,7 +24,7 @@ class VPS(models.Model):
verbose_name = "VPS"
verbose_name_plural = "VPSs"
def __unicode__(self):
def __str__(self):
return self.hostname
def set_password(self, raw_password):

3
orchestra/apps/webapps/admin.py
View File

@ -1,6 +1,7 @@
from django import forms
from django.contrib import admin
from django.core.urlresolvers import reverse
from django.utils.encoding import force_text
from django.utils.translation import ugettext, ugettext_lazy as _
from orchestra.admin import ExtendedModelAdmin
@ -20,7 +21,7 @@ class WebAppOptionInline(admin.TabularInline):
extra = 1
OPTIONS_HELP_TEXT = {
op.name: str(unicode(op.help_text)) for op in AppOption.get_plugins()
op.name: force_text(op.help_text) for op in AppOption.get_plugins()
}
class Media:

7
orchestra/apps/webapps/backends/php.py
View File

@ -118,6 +118,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
listen.owner = {{ user }}
listen.group = {{ group }}
pm = ondemand
pm.max_requests = {{ max_requests }}
{% if max_children %}pm.max_children = {{ max_children }}{% endif %}
{% if request_terminate_timeout %}request_terminate_timeout = {{ request_terminate_timeout }}{% endif %}
{% for name, value in init_vars.iteritems %}
@ -131,7 +132,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
# Format PHP init vars
init_vars = opt.get_php_init_vars(merge=self.MERGE)
if init_vars:
init_vars = [ '-d %s="%s"' % (k,v) for k,v in init_vars.iteritems() ]
init_vars = [ '-d %s="%s"' % (k,v) for k,v in init_vars.items() ]
init_vars = ', '.join(init_vars)
context.update({
'php_binary': os.path.normpath(settings.WEBAPPS_PHP_CGI_BINARY_PATH % context),
@ -144,6 +145,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
# %(banner)s
export PHPRC=%(php_rc)s
export PHP_INI_SCAN_DIR=%(php_ini_scan)s
export PHP_FCGI_MAX_REQUESTS=%(max_requests)s
exec %(php_binary)s %(php_init_vars)s""") % context
def get_fcgid_cmd_options(self, webapp, context):
@ -152,7 +154,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
'IOTimeout': webapp.get_options().get('timeout', None),
}
cmd_options = []
for directive, value in maps.iteritems():
for directive, value in maps.items():
if value:
cmd_options.append("%s %s" % (directive, value))
if cmd_options:
@ -187,6 +189,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
context.update({
'php_version': webapp.type_instance.get_php_version(),
'php_version_number': webapp.type_instance.get_php_version_number(),
'max_requests': settings.WEBAPPS_PHP_MAX_REQUESTS,
})
self.update_fcgid_context(webapp, context)
self.update_fpm_context(webapp, context)

1
orchestra/apps/webapps/migrations/0001_initial.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import orchestra.core.validators

1
orchestra/apps/webapps/migrations/0002_webapp_data.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import jsonfield.fields

1
orchestra/apps/webapps/migrations/0003_auto_20150310_2103.py
View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations

5
orchestra/apps/webapps/models.py
View File

@ -1,3 +1,4 @@
import os
import re
@ -37,7 +38,7 @@ class WebApp(models.Model):
verbose_name = _("Web App")
verbose_name_plural = _("Web Apps")
def __unicode__(self):
def __str__(self):
return self.name
def get_description(self):
@ -98,7 +99,7 @@ class WebAppOption(models.Model):
verbose_name = _("option")
verbose_name_plural = _("options")
def __unicode__(self):
def __str__(self):
return self.name
@cached_property

9
orchestra/apps/webapps/settings.py
View File

@ -30,6 +30,13 @@ WEBAPPS_FCGID_CMD_OPTIONS_PATH = getattr(settings, 'WEBAPPS_FCGID_CMD_OPTIONS_PA
)
# Greater or equal to your FcgidMaxRequestsPerProcess
# http://httpd.apache.org/mod_fcgid/mod/mod_fcgid.html#examples
WEBAPPS_PHP_MAX_REQUESTS = getattr(settings, 'WEBAPPS_PHP_MAX_REQUESTS',
400
)
WEBAPPS_PHP_ERROR_LOG_PATH = getattr(settings, 'WEBAPPS_PHP_ERROR_LOG_PATH',
''
)
@ -92,7 +99,7 @@ WEBAPPS_UNDER_CONSTRUCTION_PATH = getattr(settings, 'WEBAPPS_UNDER_CONSTRUCTION_
#WEBAPPS_TYPES_OVERRIDE = getattr(settings, 'WEBAPPS_TYPES_OVERRIDE', {})
#for webapp_type, value in WEBAPPS_TYPES_OVERRIDE.iteritems():
#for webapp_type, value in WEBAPPS_TYPES_OVERRIDE.items():
# if value is None:
# WEBAPPS_TYPES.pop(webapp_type, None)
# else:

2
orchestra/apps/webapps/tests/functional_tests/tests.py
View File

@ -2,7 +2,7 @@ import ftplib
import os
import time
import textwrap
from StringIO import StringIO
from io import StringIO
from django.conf import settings as djsettings
from django.contrib.contenttypes.models import ContentType

7
orchestra/apps/webapps/types/php.py
View File

@ -61,7 +61,7 @@ class PHPApp(AppType):
@cached
def get_php_options(self):
php_version = self.get_php_version()
php_version = self.get_php_version_number()
php_options = AppOption.get_option_groups()[AppOption.PHP]
return [op for op in php_options if getattr(self, 'deprecated', 999) > php_version]
@ -93,6 +93,9 @@ class PHPApp(AppType):
if function not in enabled_functions:
disabled_functions.append(function)
init_vars['dissabled_functions'] = ','.join(disabled_functions)
timeout = self.instance.options.filter(name='timeout').first()
if timeout:
init_vars['max_execution_time'] = timeout.value
if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars:
context = self.get_directive_context()
error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context)
@ -128,4 +131,4 @@ class PHPApp(AppType):
raise ValueError("No version number matches for '%s'" % php_version)
if len(number) > 1:
raise ValueError("Multiple version number matches for '%s'" % php_version)
return number[0]
return float(number[0])

3
orchestra/apps/websites/admin.py
View File

@ -2,6 +2,7 @@ from django import forms
from django.contrib import admin
from django.core.urlresolvers import resolve
from django.db.models import Q
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
@ -22,7 +23,7 @@ class WebsiteDirectiveInline(admin.TabularInline):
extra = 1
DIRECTIVES_HELP_TEXT = {
op.name: str(unicode(op.help_text)) for op in SiteDirective.get_plugins()
op.name: force_text(op.help_text) for op in SiteDirective.get_plugins()
}
def formfield_for_dbfield(self, db_field, **kwargs):

2
orchestra/apps/websites/backends/apache.py
View File

@ -219,7 +219,7 @@ class Apache2Backend(ServiceController):
def get_saas(self, directives):
saas = []
for name, values in directives.iteritems():
for name, values in directives.items():
if name.endswith('-saas'):
for value in values:
context = {

2
orchestra/apps/websites/directives.py
View File

@ -47,7 +47,7 @@ class SiteDirective(Plugin):
options = cls.get_option_groups()
for option in options.pop(None, ()):
yield (option.name, option.verbose_name)
for group, options in options.iteritems():
for group, options in options.items():
yield (group, [(op.name, op.verbose_name) for op in options])
def validate(self, website):

3
orchestra/apps/websites/forms.py
View File

@ -1,5 +1,6 @@
from django import forms
from django.core.exceptions import ValidationError
from django.utils.encoding import force_text
from .validators import validate_domain_protocol
@ -36,7 +37,7 @@ class WebsiteDirectiveInlineFormSet(forms.models.BaseInlineFormSet):
if value is not None:
if directive.unique_value and value in values.get(name, []):
form.add_error('value', ValidationError(
_("This value is already used by other %s.") % unicode(directive.get_verbose_name())
_("This value is already used by other %s.") % force_text(directive.get_verbose_name())
))
try:
values[name].append(value)

6
orchestra/apps/websites/models.py
View File

@ -40,7 +40,7 @@ class Website(models.Model):
class Meta:
unique_together = ('name', 'account')
def __unicode__(self):
def __str__(self):
return self.name
@property
@ -107,7 +107,7 @@ class WebsiteDirective(models.Model):
choices=SiteDirective.get_choices())
value = models.CharField(_("value"), max_length=256)
def __unicode__(self):
def __str__(self):
return self.name
@cached_property
@ -133,7 +133,7 @@ class Content(models.Model):
class Meta:
unique_together = ('website', 'path')
def __unicode__(self):
def __str__(self):
try:
return self.website.name + self.path
except Website.DoesNotExist:

12
orchestra/bin/orchestra-admin
View File

@ -136,10 +136,10 @@ function install_requirements () {
PIP="django==1.7.7 \
django-celery-email==1.0.4 \
django-fluent-dashboard==0.3.5 \
django-fluent-dashboard==0.4 \
https://bitbucket.org/izi/django-admin-tools/get/a0abfffd76a0.zip \
IPy==0.81 \
django-extensions==1.1.1 \
django-extensions==1.5.2 \
django-transaction-signals==1.0.0 \
django-celery==3.1.16 \
celery==3.1.16 \
@ -209,10 +209,10 @@ function install_requirements () {
# Patch passlib
IMPORT="from django.contrib.auth.hashers import mask_hash, _"
COLLECTIONS="from collections import OrderedDict"
sed -i "s/${IMPORT}, SortedDict/${IMPORT}\n ${COLLECTIONS}/" \
/usr/local/lib/python2.7/dist-packages/passlib/ext/django/utils.py
sed -i "s/SortedDict/OrderedDict/g" \
/usr/local/lib/python2.7/dist-packages/passlib/ext/django/utils.py
ls /usr/local/lib/python*/dist-packages/passlib/ext/django/utils.py \
| xargs sed -i "s/${IMPORT}, SortedDict/${IMPORT}\n ${COLLECTIONS}/"
ls /usr/local/lib/python*/dist-packages/passlib/ext/django/utils.py \
| xargs sed -i "s/SortedDict/OrderedDict/g"
# Patch dateutil
sed -i "s/elif not isinstance(dt2, datetime.datetime):/else:/" \

3
orchestra/conf/base_settings.py
View File

@ -48,8 +48,7 @@ MIDDLEWARE_CLASSES = (
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'orchestra.core.caches.RequestCacheMiddleware',
# ATOMIC REQUESTS do not wrap middlewares
'orchestra.core.middlewares.TransactionMiddleware',
# also handles transations, ATOMIC REQUESTS does not wrap middlewares
'orchestra.apps.orchestration.middlewares.OperationsMiddleware',
# Uncomment the next line for simple clickjacking protection:
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',

74
orchestra/core/middlewares.py
View File

@ -8,40 +8,40 @@ class TransactionMiddleware(object):
commit, the commit is done when a successful response is created. If an
exception happens, the database is rolled back.
"""
def process_request(self, request):
"""Enters transaction management"""
transaction.enter_transaction_management()
def process_exception(self, request, exception):
"""Rolls back the database and leaves transaction management"""
if transaction.is_dirty():
# This rollback might fail because of network failure for example.
# If rollback isn't possible it is impossible to clean the
# connection's state. So leave the connection in dirty state and
# let request_finished signal deal with cleaning the connection.
transaction.rollback()
transaction.leave_transaction_management()
def process_response(self, request, response):
"""Commits and leaves transaction management."""
if not transaction.get_autocommit():
if transaction.is_dirty():
# Note: it is possible that the commit fails. If the reason is
# closed connection or some similar reason, then there is
# little hope to proceed nicely. However, in some cases (
# deferred foreign key checks for exampl) it is still possible
# to rollback().
try:
transaction.commit()
except Exception:
# If the rollback fails, the transaction state will be
# messed up. It doesn't matter, the connection will be set
# to clean state after the request finishes. And, we can't
# clean the state here properly even if we wanted to, the
# connection is in transaction but we can't rollback...
transaction.rollback()
transaction.leave_transaction_management()
raise
transaction.leave_transaction_management()
return response
pass
# def process_request(self, request):
# """Enters transaction management"""
# transaction.enter_transaction_management()
#
# def process_exception(self, request, exception):
# """Rolls back the database and leaves transaction management"""
# if transaction.is_dirty():
# # This rollback might fail because of network failure for example.
# # If rollback isn't possible it is impossible to clean the
# # connection's state. So leave the connection in dirty state and
# # let request_finished signal deal with cleaning the connection.
# transaction.rollback()
# transaction.leave_transaction_management()
#
# def process_response(self, request, response):
# """Commits and leaves transaction management."""
# if not transaction.get_autocommit():
# if transaction.is_dirty():
# # Note: it is possible that the commit fails. If the reason is
# # closed connection or some similar reason, then there is
# # little hope to proceed nicely. However, in some cases (
# # deferred foreign key checks for exampl) it is still possible
# # to rollback().
# try:
# transaction.commit()
# except Exception:
# # If the rollback fails, the transaction state will be
# # messed up. It doesn't matter, the connection will be set
# # to clean state after the request finishes. And, we can't
# # clean the state here properly even if we wanted to, the
# # connection is in transaction but we can't rollback...
# transaction.rollback()
# transaction.leave_transaction_management()
# raise
# transaction.leave_transaction_management()
# return response

2
orchestra/core/validators.py
View File

@ -15,7 +15,7 @@ from ..utils.python import import_class
def all_valid(kwargs):
""" helper function to merge multiple validators at once """
errors = {}
for field, validator in kwargs.iteritems():
for field, validator in kwargs.items():
try:
validator[0](*validator[1:])
except ValidationError as error:

6
orchestra/forms/widgets.py
View File

@ -19,16 +19,16 @@ class ShowTextWidget(forms.Widget):
if hasattr(self, 'initial'):
value = self.initial
if self.bold:
final_value = u'<b>%s</b>' % (value)
final_value = '<b>%s</b>' % (value)
else:
final_value = '<br/>'.join(value.split('\n'))
if self.warning:
final_value = (
u'<ul class="messagelist"><li class="warning">%s</li></ul>'
'<ul class="messagelist"><li class="warning">%s</li></ul>'
% final_value)
if self.hidden:
final_value = (
u'%s<input type="hidden" name="%s" value="%s"/>'
'%s<input type="hidden" name="%s" value="%s"/>'
% (final_value, name, value))
return mark_safe(final_value)

10
orchestra/management/commands/makemessages.py
View File

@ -18,13 +18,13 @@ class Command(makemessages.Command):
self.remove_database_files()
def get_contents(self):
for model, fields in ModelTranslation._registry.iteritems():
for model, fields in ModelTranslation._registry.items():
for field in fields:
contents = []
for content in model.objects.values_list('id', field):
pk, value = content
contents.append(
(pk, u"_(u'%s')" % value)
(pk, "_(u'%s')" % value)
)
if contents:
yield ('_'.join((model._meta.db_table, field)), contents)
@ -38,7 +38,7 @@ class Command(makemessages.Command):
2) Django's makemessages will work with no modifications
"""
for name, contents in self.get_contents():
name = unicode(name)
name = str(name)
maximum = None
content = {}
for pk, value in contents:
@ -46,9 +46,9 @@ class Command(makemessages.Command):
maximum = pk
content[pk] = value
tmpcontent = []
for ix in xrange(maximum+1):
for ix in range(maximum+1):
tmpcontent.append(content.get(ix, ''))
tmpcontent = u'\n'.join(tmpcontent) + '\n'
tmpcontent = '\n'.join(tmpcontent) + '\n'
filename = 'database_%s.sql.py' % name
self.database_files.append(filename)
with open(filename, 'w') as tmpfile:

Some files were not shown because too many files have changed in this diff Show More