pangea/django-orchestra
7
0
Fork 0
Code Issues 1 Pull requests 2 Packages Projects Releases Wiki Activity

Refactored SaaS application

Browse source
This commit is contained in:
Marc Aymerich 2014-11-20 15:34:59 +00:00
parent 50c2397924
commit 917b0b9329
20 changed files with 243 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
TODO.md
View file

@ -181,3 +181,26 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* Move plugins back from apps to orchestra main app
* BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when)
* rename admin prefetch_related to list_prefetch_related for consistency
* LAST resource monitor option -> SUM(last backend)
* Resource.monitor(async=True) admin action
* Validate a model path exists between resource.content_type and backend.model
* Add support for whitelisted IPs on traffic monitoring ['127.0.0.1',]
* Periodic task for cleaning old monitoring data
* Generate reports of Account contracted services
* Create an admin service_view with icons (like SaaS app)
* Fix ftp traffic
* Resource graph for each related object

1
orchestra/apps/contacts/admin.py
View file

@ -14,6 +14,7 @@ class ContactAdmin(AccountAdminMixin, admin.ModelAdmin):
list_display = (
'dispaly_name', 'email', 'phone', 'phone2', 'country', 'account_link'
)
# TODO email usage custom filter contains
list_filter = ('email_usage',)
search_fields = (
'contact__account__name', 'short_name', 'full_name', 'phone', 'phone2',

1
orchestra/apps/lists/admin.py
View file

@ -46,6 +46,7 @@ class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModel
change_readonly_fields = ('name',)
form = ListChangeForm
add_form = ListCreationForm
list_select_related = ('account', 'address_domain',)
filter_by_account_fields = ['address_domain']
address_domain_link = admin_link('address_domain', order='address_domain__name')

2
orchestra/apps/mailboxes/admin.py
View file

@ -96,7 +96,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo
class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
list_display = (
'email', 'domain_link', 'display_mailboxes', 'display_forward', 'account_link'
'email', 'account_link', 'domain_link', 'display_mailboxes', 'display_forward',
)
list_filter = (HasMailboxListFilter, HasForwardListFilter)
fields = ('account_link', ('name', 'domain'), 'mailboxes', 'forward')

1
orchestra/apps/resources/admin.py
View file

@ -102,6 +102,7 @@ class ResourceDataAdmin(ExtendedModelAdmin):
resource_link = admin_link('resource')
content_object_link = admin_link('content_object')
content_object_link.admin_order_field = None
display_updated = admin_date('updated_at', short_description=_("Updated"))
def get_urls(self):

19
orchestra/apps/saas/admin.py
View file

@ -1,4 +1,5 @@
from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.accounts.admin import AccountAdminMixin
from orchestra.apps.plugins.admin import SelectPluginAdminMixin
@ -8,10 +9,26 @@ from .services import SoftwareService
class SaaSAdmin(SelectPluginAdminMixin, AccountAdminMixin, admin.ModelAdmin):
list_display = ('description', 'service', 'account_link')
list_display = ('username', 'service', 'display_site_name', 'account_link')
list_filter = ('service',)
plugin = SoftwareService
plugin_field = 'service'
def display_site_name(self, saas):
site_name = saas.get_site_name()
return '<a href="http://%s">%s</a>' % (site_name, site_name)
display_site_name.short_description = _("Site name")
display_site_name.allow_tags = True
display_site_name.admin_order_field = 'site_name'
def get_fields(self, request, obj=None):
fields = super(SaaSAdmin, self).get_fields(request, obj)
fields = list(fields)
# TODO do it in AccountAdminMixin?
if obj is not None:
fields.remove('account')
else:
fields.remove('account_link')
return fields
admin.site.register(SaaS, SaaSAdmin)

28
orchestra/apps/saas/models.py
View file

@ -3,7 +3,8 @@ from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from jsonfield import JSONField
from orchestra.core import services
from orchestra.core import services, validators
from orchestra.models.fields import NullableCharField
from .services import SoftwareService
@ -11,33 +12,36 @@ from .services import SoftwareService
class SaaS(models.Model):
service = models.CharField(_("service"), max_length=32,
choices=SoftwareService.get_plugin_choices())
# TODO use model username password instead of data
# username = models.CharField(_("username"), max_length=64, unique=True,
# help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."),
# validators=[validators.RegexValidator(r'^[\w.-]+$',
# _("Enter a valid username."), 'invalid')])
# password = models.CharField(_("password"), max_length=128)
username = models.CharField(_("username"), max_length=64,
help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."),
validators=[validators.validate_username])
site_name = NullableCharField(_("site name"), max_length=32, null=True)
account = models.ForeignKey('accounts.Account', verbose_name=_("account"),
related_name='saas')
data = JSONField(_("data"))
data = JSONField(_("data"), help_text=_("Extra information dependent of each service."))
class Meta:
verbose_name = "SaaS"
verbose_name_plural = "SaaS"
unique_together = (
('username', 'service'),
('site_name', 'service'),
)
def __unicode__(self):
return "%s (%s)" % (self.description, self.service_class.verbose_name)
return "%s@%s" % (self.username, self.service)
@cached_property
def service_class(self):
return SoftwareService.get_plugin(self.service)
@cached_property
def description(self):
return self.service_class().get_description(self.data)
def get_site_name(self):
return self.service_class().get_site_name(self)
def clean(self):
self.data = self.service_class().clean_data(self)
def set_password(self, password):
self.password = password
services.register(SaaS)

41
orchestra/apps/saas/services/bscw.py
View file

@ -3,51 +3,26 @@ from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from orchestra.apps.plugins.forms import PluginDataForm
from orchestra.core import validators
from .options import SoftwareService
from .options import SoftwareService, SoftwareServiceForm
# TODO monitor quota since out of sync?
class BSCWForm(PluginDataForm):
username = forms.CharField(label=_("Username"), max_length=64)
password = forms.CharField(label=_("Password"), max_length=256, required=False)
email = forms.EmailField(label=_("Email"))
class BSCWForm(SoftwareServiceForm):
email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'}))
quota = forms.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB."))
class SEPADirectDebitSerializer(serializers.Serializer):
username = serializers.CharField(label=_("Username"), max_length=64,
validators=[validators.validate_name])
password = serializers.CharField(label=_("Password"), max_length=256, required=False,
write_only=True)
class BSCWDataSerializer(serializers.Serializer):
email = serializers.EmailField(label=_("Email"))
quota = serializers.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB."))
def validate(self, data):
data['username'] = data['username'].strip()
return data
class BSCWService(SoftwareService):
verbose_name = "BSCW"
form = BSCWForm
serializer = SEPADirectDebitSerializer
description_field = 'username'
serializer = BSCWDataSerializer
icon = 'saas/icons/BSCW.png'
@classmethod
def clean_data(cls, saas):
try:
data = super(BSCWService, cls).clean_data(saas)
except ValidationError, error:
if not saas.pk and 'password' not in saas.data:
error.error_dict['password'] = [_("Password is required.")]
raise error
if not saas.pk and 'password' not in saas.data:
raise ValidationError({
'password': _("Password is required.")
})
return data
# TODO override from settings
site_name = 'bascw.orchestra.lan'
change_readonly_fileds = ('email',)

14
orchestra/apps/saas/services/dokuwiki.py
View file

@ -1,20 +1,6 @@
from django import forms
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.plugins.forms import PluginDataForm
from .options import SoftwareService
class DowkuwikiForm(PluginDataForm):
username = forms.CharField(label=_("Username"), max_length=64)
password = forms.CharField(label=_("Password"), max_length=64)
site_name = forms.CharField(label=_("Site name"), max_length=64)
email = forms.EmailField(label=_("Email"))
class DokuwikiService(SoftwareService):
verbose_name = "Dowkuwiki"
form = DowkuwikiForm
description_field = 'site_name'
icon = 'saas/icons/Dokuwiki.png'

14
orchestra/apps/saas/services/drupal.py
View file

@ -1,20 +1,6 @@
from django import forms
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.plugins.forms import PluginDataForm
from .options import SoftwareService
class DrupalForm(PluginDataForm):
username = forms.CharField(label=_("Username"), max_length=64)
password = forms.CharField(label=_("Password"), max_length=64)
site_name = forms.CharField(label=_("Site name"), max_length=64)
email = forms.EmailField(label=_("Email"))
class DrupalService(SoftwareService):
verbose_name = "Drupal"
form = DrupalForm
description_field = 'site_name'
icon = 'saas/icons/Drupal.png'

14
orchestra/apps/saas/services/gitlab.py
View file

@ -1,20 +1,6 @@
from django import forms
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.plugins.forms import PluginDataForm
from .options import SoftwareService
class GitLabForm(PluginDataForm):
username = forms.CharField(label=_("Username"), max_length=64)
password = forms.CharField(label=_("Password"), max_length=64)
project_name = forms.CharField(label=_("Project name"), max_length=64)
email = forms.EmailField(label=_("Email"))
class GitLabService(SoftwareService):
verbose_name = "GitLab"
form = GitLabForm
description_field = 'project_name'
icon = 'saas/icons/gitlab.png'

84
orchestra/apps/saas/services/options.py
View file

@ -1,21 +1,84 @@
from django import forms
from django.core.exceptions import ValidationError
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from orchestra.apps import plugins
from orchestra.apps.plugins.forms import PluginDataForm
from orchestra.core import validators
from orchestra.forms import widgets
from orchestra.utils.functional import cached
from orchestra.utils.python import import_class
from .. import settings
# TODO if unique_description: make description_field create only
class SoftwareServiceForm(PluginDataForm):
password = forms.CharField(label=_("Password"), required=False,
widget=widgets.ReadOnlyWidget('<strong>Unknown password</strong>'),
help_text=_("Servide passwords are not stored, so there is no way to see this "
"service's password, but you can change the password using "
"<a href=\"password/\">this form</a>."))
password1 = forms.CharField(label=_("Password"), validators=[validators.validate_password],
widget=forms.PasswordInput)
password2 = forms.CharField(label=_("Password confirmation"),
widget=forms.PasswordInput,
help_text=_("Enter the same password as above, for verification."))
def __init__(self, *args, **kwargs):
super(SoftwareServiceForm, self).__init__(*args, **kwargs)
self.is_change = bool(self.instance and self.instance.pk)
if self.is_change:
for field in self.plugin.change_readonly_fileds + ('username',):
value = getattr(self.instance, field, None) or self.instance.data[field]
self.fields[field].required = False
self.fields[field].widget = widgets.ReadOnlyWidget(value)
self.fields[field].help_text = None
site_name = self.instance.get_site_name()
self.fields['password1'].required = False
self.fields['password1'].widget = forms.HiddenInput()
self.fields['password2'].required = False
self.fields['password2'].widget = forms.HiddenInput()
else:
self.fields['password'].widget = forms.HiddenInput()
site_name = self.plugin.site_name
if site_name:
link = '<a href="http://%s">%s</a>' % (site_name, site_name)
self.fields['site_name'].widget = widgets.ReadOnlyWidget(site_name, mark_safe(link))
self.fields['site_name'].required = False
else:
base_name = self.plugin.site_name_base_domain
help_text = _("The final URL would be &lt;site_name&gt;.%s") % base_name
self.fields['site_name'].help_text = help_text
def clean_password2(self):
if not self.is_change:
password1 = self.cleaned_data.get("password1")
password2 = self.cleaned_data.get("password2")
if password1 and password2 and password1 != password2:
msg = _("The two password fields didn't match.")
raise forms.ValidationError(msg)
return password2
def clean_site_name(self):
if self.plugin.site_name:
return None
return self.cleaned_data['site_name']
def save(self, commit=True):
obj = super(SoftwareServiceForm, self).save(commit=commit)
if not self.is_change:
obj.set_password(self.cleaned_data["password1"])
return obj
class SoftwareService(plugins.Plugin):
description_field = ''
unique_description = True
form = None
form = SoftwareServiceForm
serializer = None
site_name = None
site_name_base_domain = 'orchestra.lan'
icon = 'orchestra/icons/apps.png'
change_readonly_fileds = ()
class_verbose_name = _("Software as a Service")
@classmethod
@ -29,18 +92,14 @@ class SoftwareService(plugins.Plugin):
@classmethod
def clean_data(cls, saas):
""" model clean, uses cls.serizlier by default """
if cls.unique_description and not saas.pk:
from ..models import SaaS
field = cls.description_field
if SaaS.objects.filter(data__contains='"%s":"%s"' % (field, saas.data[field])).exists():
raise ValidationError({
field: _("SaaS service with this %(field)s already exists.")
}, params={'field': field})
serializer = cls.serializer(data=saas.data)
if not serializer.is_valid():
raise ValidationError(serializer.errors)
return serializer.data
def get_site_name(self, saas):
return self.site_name or '.'.join((saas.site_name, self.site_name_base_domain))
def get_form(self):
self.form.plugin = self
self.form.plugin_field = 'service'
@ -49,6 +108,3 @@ class SoftwareService(plugins.Plugin):
def get_serializer(self):
self.serializer.plugin = self
return self.serializer
def get_description(self, data):
return data[self.description_field]

9
orchestra/apps/saas/services/phplist.py
View file

@ -1,17 +1,14 @@
from django import forms
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.plugins.forms import PluginDataForm
from .options import SoftwareService
from .options import SoftwareService, SoftwareServiceForm
class PHPListForm(PluginDataForm):
email = forms.EmailField(label=_("Email"))
class PHPListForm(SoftwareServiceForm):
email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'}))
class PHPListService(SoftwareService):
verbose_name = "phpList"
form = PHPListForm
description_field = 'email'
icon = 'saas/icons/Phplist.png'

27
orchestra/apps/saas/services/wordpress.py
View file

@ -1,30 +1,23 @@
from django import forms
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from orchestra.apps.plugins.forms import PluginDataForm
from .options import SoftwareService
from .options import SoftwareService, SoftwareServiceForm
class WordPressForm(PluginDataForm):
username = forms.CharField(label=_("Username"), max_length=64)
password = forms.CharField(label=_("Password"), max_length=64)
site_name = forms.CharField(label=_("Site name"), max_length=64,
help_text=_("URL will be &lt;site_name&gt;.blogs.orchestra.lan"))
email = forms.EmailField(label=_("Email"))
class WordPressForm(SoftwareServiceForm):
email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'}))
def __init__(self, *args, **kwargs):
super(WordPressForm, self).__init__(*args, **kwargs)
instance = kwargs.get('instance')
if instance:
url = 'http://%s.%s' % (instance.data['site_name'], 'blogs.orchestra.lan')
url = '<a href="%s">%s</a>' % (url, url)
self.fields['site_name'].help_text = mark_safe(url)
class WordPressDataSerializer(serializers.Serializer):
email = serializers.EmailField(label=_("Email"))
class WordPressService(SoftwareService):
verbose_name = "WordPress"
form = WordPressForm
description_field = 'site_name'
serializer = WordPressDataSerializer
icon = 'saas/icons/WordPress.png'
site_name_base_domain = 'blogs.orchestra.lan'
change_readonly_fileds = ('email',)

17
orchestra/apps/systemusers/backends.py
View file

@ -99,19 +99,21 @@ class SystemUserDisk(ServiceMonitor):
class FTPTraffic(ServiceMonitor):
model = 'systemusers.SystemUser'
resource = ServiceMonitor.TRAFFIC
verbose_name = _('Main FTP traffic')
verbose_name = _('Systemuser FTP traffic')
def prepare(self):
current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z")
self.append(textwrap.dedent("""\
function monitor () {
OBJECT_ID=$1
INI_DATE=$2
INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2")
END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s')
USERNAME="$3"
LOG_FILE="$4"
{
grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \\
| grep " \\[${USERNAME}\\] " \\
| awk -v ini="${INI_DATE}" end="$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s')" '
| awk -v ini="${INI_DATE}" -v end="${END_DATE}" '
BEGIN {
sum = 0
months["Jan"] = "01"
@ -131,20 +133,21 @@ class FTPTraffic(ServiceMonitor):
split($4, t, ":")
# line_date = year month day hour minute second
line_date = $5 months[$2] $3 t[1] t[2] t[3]
if ( line_date > ini && line_date < end)
if ( line_date > ini && line_date < end) {
split($0, l, "\\", ")
split(l[3], b, " ")
sum += b[1]
}
} END {
print sum
}
' | xargs echo ${OBJECT_ID}
}' || [[ $? == 1 ]] && true
} | xargs echo ${OBJECT_ID}
}""" % current_date))
def monitor(self, user):
context = self.get_context(user)
self.append(
'monitor %{object_id} $(date "+%Y%m%d%H%M%S" -d "{last_date}") "{username}" "{log_file}"'.format(**context)
'monitor {object_id} "{last_date}" "{username}" {log_file}'.format(**context)
)
def get_context(self, user):

7
orchestra/apps/systemusers/models.py
View file

@ -1,14 +1,13 @@
import os
from django.contrib.auth.hashers import make_password
from django.core import validators
from django.core.exceptions import ValidationError
from django.core.mail import send_mail
from django.db import models
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from orchestra.core import services
from orchestra.core import services, validators
from . import settings
@ -25,9 +24,7 @@ class SystemUser(models.Model):
""" System users """
username = models.CharField(_("username"), max_length=64, unique=True,
help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."),
validators=[
validators.RegexValidator(r'^[\w.-]+$', _("Enter a valid username."))
])
validators=[validators.validate_username])
password = models.CharField(_("password"), max_length=128)
account = models.ForeignKey('accounts.Account', verbose_name=_("Account"),
related_name='systemusers')

18
orchestra/apps/websites/backends/apache.py
View file

@ -218,14 +218,20 @@ class Apache2Traffic(ServiceMonitor):
verbose_name = _("Apache 2 Traffic")
def prepare(self):
current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z")
ignore_hosts = '\\|'.join(settings.WEBSITES_TRAFFIC_IGNORE_HOSTS)
context = {
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'ignore_hosts': '-v "%s"' % ignore_hosts if ignore_hosts else '',
}
self.append(textwrap.dedent("""\
function monitor () {
OBJECT_ID=$1
INI_DATE=$2
INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2")
END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s')
LOG_FILE="$3"
{
awk -v ini="${INI_DATE}" -v end="$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s')" '
{ grep "%(ignore_hosts)s" "${LOG_FILE}" || echo '\\n'; } \\
| awk -v ini="${INI_DATE}" -v end="${END_DATE}" '
BEGIN {
sum = 0
months["Jan"] = "01";
@ -254,13 +260,13 @@ class Apache2Traffic(ServiceMonitor):
sum += $NF
} END {
print sum
}' "${LOG_FILE}" || echo 0
}' || [[ $? == 1 ]] && true
} | xargs echo ${OBJECT_ID}
}""" % current_date))
}""" % context))
def monitor(self, site):
context = self.get_context(site)
self.append('monitor {object_id} $(date "+%Y%m%d%H%M%S" -d "{last_date}") {log_file}'.format(**context))
self.append('monitor {object_id} "{last_date}" {log_file}'.format(**context))
def get_context(self, site):
return {

4
orchestra/apps/websites/settings.py
View file

@ -84,3 +84,7 @@ WEBSITES_WEBALIZER_PATH = getattr(settings, 'WEBSITES_WEBALIZER_PATH',
WEBSITES_WEBSITE_WWW_LOG_PATH = getattr(settings, 'WEBSITES_WEBSITE_WWW_LOG_PATH',
# %(user_home)s %(name)s %(unique_name)s %(username)s
'/var/log/apache2/virtual/%(unique_name)s')
WEBSITES_TRAFFIC_IGNORE_HOSTS = getattr(settings, 'WEBSITES_TRAFFIC_IGNORE_HOSTS',
[])

4
orchestra/conf/base_settings.py
View file

@ -162,7 +162,7 @@ FLUENT_DASHBOARD_APP_GROUPS = (
'orchestra.apps.orchestration.models.BackendLog',
'orchestra.apps.orchestration.models.Server',
'orchestra.apps.resources.models.Resource',
'orchestra.apps.resources.models.Monitor',
'orchestra.apps.resources.models.ResourceData',
'orchestra.apps.services.models.Service',
'orchestra.apps.plans.models.Plan',
'orchestra.apps.miscellaneous.models.MiscService',
@ -206,7 +206,7 @@ FLUENT_DASHBOARD_APP_ICONS = {
'orchestration/route': 'hal.png',
'orchestration/backendlog': 'scriptlog.png',
'resources/resource': "gauge.png",
'resources/monitor': "Utilities-system-monitor.png",
'resources/resourcedata': "monitor.png",
'plans/plan': 'Pack.png',
}

4
orchestra/core/validators.py
View file

@ -84,6 +84,10 @@ def validate_hostname(hostname):
raise ValidationError(_("Not a valid hostname (%s).") % name)
def validate_username(value):
validators.RegexValidator(r'^[\w.-]+$', _("Enter a valid username."))(value)
def validate_password(value):
try:
crack.VeryFascistCheck(value)