Added support for multiple php fpm webapps

TODO.md

@@ -427,3 +427,6 @@ serailzer self.instance on create.
 
 
 # IF modsecurity... and Merge websites locations
+# backend email error log with links to instances
+# PHP backend multiple FPM directories support
+

orchestra/contrib/domains/backends.py

@@ -111,7 +111,7 @@ class Bind9MasterDomainBackend(ServiceController):
         from orchestra.contrib.orchestration.manager import router
         operation = Operation(backend, domain, Operation.SAVE)
         servers = []
-        for routes in router.get_routes(operation):
+        for route in router.get_routes(operation):
             servers.append(route.host.get_ip())
         return servers
     
@@ -125,6 +125,7 @@ class Bind9MasterDomainBackend(ServiceController):
         ips = []
         masters_ips = self.get_masters_ips(domain)
         records = domain.get_records()
+        # Slaves from NS
         for record in records.by_type(Record.NS):
             hostname = record.value.rstrip('.')
             # First try with a DNS query, a more reliable source
@@ -141,6 +142,10 @@ class Bind9MasterDomainBackend(ServiceController):
                     addr = records.by_type(Record.A)[0].value
             if addr not in masters_ips:
                 ips.append(addr)
+        # Slaves from internal networks
+        if not settings.DOMAINS_MASTERS:
+            for server in self.get_servers(domain, Bind9SlaveDomainBackend):
+                ips.append(server)
         return OrderedSet(sorted(ips))
     
     def get_context(self, domain):

orchestra/contrib/orchestration/helpers.py

@@ -2,11 +2,14 @@ import textwrap
 
 from django.contrib import messages
 from django.core.mail import mail_admins
-from django.core.urlresolvers import reverse
+from django.core.urlresolvers import reverse, NoReverseMatch
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 from django.utils.translation import ungettext, ugettext_lazy as _
 
+from orchestra import settings as orchestra_settings
+from orchestra.admin.utils import change_url
+
 
 def get_backends_help_text(backends):
     help_texts = {}
@@ -44,17 +47,29 @@ def get_backends_help_text(backends):
     return help_texts
 
 
+def get_instance_url(operation):
+    try:
+        url = change_url(operation.instance)
+    except NoReverseMatch:
+        return _("Deleted {0}").format(operation.instance_repr or '-'.join(
+            (escape(operation.content_type), escape(operation.object_id))))
+    return orchestra_settings.ORCHESTRA_SITE_URL + url
+
+
 def send_report(method, args, log):
     server = args[0]
     backend = method.__self__.__class__.__name__
     subject = '[Orchestra] %s execution %s on %s'  % (backend, log.state, server)
     separator = "\n%s\n\n" % ('~ '*40,)
+    print(log.operations.all())
+    operations = '\n'.join([' '.join((op.action, get_instance_url(op))) for op in log.operations.all()])
     message = separator.join([
         "[EXIT CODE] %s" % log.exit_code,
         "[STDERR]\n%s" % log.stderr,
         "[STDOUT]\n%s" % log.stdout,
         "[SCRIPT]\n%s" % log.script,
         "[TRACEBACK]\n%s" % log.traceback,
+        "[OPERATIONS]\n%s" % operations,
     ])
     html_message = '\n\n'.join([
         '<h4 style="color:#505050;">Exit code %s</h4>' % log.exit_code,
@@ -66,6 +81,8 @@ def send_report(method, args, log):
             '<pre style="margin-left:20px;font-size:11px">%s</pre>' % escape(log.script),
         '<h4 style="color:#505050;">Traceback</h4>'
             '<pre style="margin-left:20px;font-size:11px">%s</pre>' % escape(log.traceback),
+        '<h4 style="color:#505050;">Operations</h4>'
+            '<pre style="margin-left:20px;font-size:11px">%s</pre>' % escape(operations),
     ])
     mail_admins(subject, message, html_message=html_message)
 
@@ -78,6 +95,7 @@ def get_backend_url(ids):
         return url + '?id__in=%s' % ','.join(map(str, ids))
     return ''
 
+
 def message_user(request, logs):
     total, successes, async = 0, 0, 0
     ids = []

orchestra/contrib/orchestration/manager.py

@@ -27,8 +27,6 @@ def keep_log(execute, log, operations):
         log = kwargs['log']
         try:
             log = execute(*args, **kwargs)
-            if not log.is_success:
-                send_report(execute, args, log)
         except Exception as e:
             trace = traceback.format_exc()
             log.state = log.EXCEPTION
@@ -44,6 +42,8 @@ def keep_log(execute, log, operations):
             for operation in operations:
                 logger.info("Executed %s" % str(operation))
                 operation.store(log)
+            if not log.is_success:
+                send_report(execute, args, log)
             stdout = log.stdout.strip()
             stdout and logger.debug('STDOUT %s', stdout)
             stderr = log.stderr.strip()

orchestra/contrib/webapps/admin.py

@@ -10,7 +10,7 @@ from orchestra.contrib.accounts.admin import AccountAdminMixin
 from orchestra.forms.widgets import DynamicHelpTextSelect
 from orchestra.plugins.admin import SelectPluginAdminMixin
 
-from .filters import HasWebsiteListFilter
+from .filters import HasWebsiteListFilter, PHPVersionListFilter
 from .models import WebApp, WebAppOption
 from .options import AppOption
 from .types import AppType
@@ -49,7 +49,7 @@ class WebAppOptionInline(admin.TabularInline):
 
 class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
     list_display = ('name', 'type', 'display_detail', 'display_websites', 'account_link')
-    list_filter = ('type', HasWebsiteListFilter)
+    list_filter = ('type', HasWebsiteListFilter, PHPVersionListFilter)
     inlines = [WebAppOptionInline]
     readonly_fields = ('account_link', )
     change_readonly_fields = ('name', 'type', 'display_websites')

orchestra/contrib/webapps/backends/php.py

@@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _
 from orchestra.contrib.orchestration import ServiceController
 
 from . import WebAppServiceMixin
-from .. import settings
+from .. import settings, utils
 
 
 class PHPBackend(WebAppServiceMixin, ServiceController):
@@ -36,10 +36,13 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
         self.create_webapp_dir(context)
         if webapp.type_instance.is_fpm:
             self.save_fpm(webapp, context)
-            self.delete_fcgid(webapp, context)
         elif webapp.type_instance.is_fcgid:
             self.save_fcgid(webapp, context)
-            self.delete_fpm(webapp, context)
+        else:
+            raise TypeError("Unknown PHP execution type")
+        # Clean php fcgid/fpm apps in order to effectively support change of php-version
+        self.delete_fcgid(webapp, context, preserve=True)
+        self.delete_fpm(webapp, context, preserve=True)
         self.set_under_construction(context)
     
     def save_fpm(self, webapp, context):
@@ -103,16 +106,39 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
             self.delete_fcgid(webapp, context)
         self.delete_webapp_dir(context)
     
-    def delete_fpm(self, webapp, context):
+    def has_sibilings(self, webapp, context):
-        # Better not delete a pool used by other apps
+        return type(webapp).objects.filter(
-        if not self.MERGE:
+            account=webapp.account_id,
-            self.append("rm -f %(fpm_path)s" % context)
+            data__contains='"php_version":"%s"' % context['php_version'],
+        ).exclude(id=webapp.pk).exists()
     
-    def delete_fcgid(self, webapp, context):
+    def delete_fpm(self, webapp, context, preserve=False):
-        # Better not delete a wrapper used by other apps
+        """ delete all pools in order to efectively support changing php-fpm version """
-        if not self.MERGE:
+        context_copy = dict(context)
-            self.append("rm -f %(wrapper_path)s" % context)
+        for php_version, verbose in settings.WEBAPPS_PHP_VERSIONS:
-            self.append("rm -f %(cmd_options_path)s" % context)
+            if preserve and php_version == context['php_version']:
+                continue
+            php_version_number = utils.extract_version_number(php_version)
+            context_copy['php_version_number'] = php_version_number
+            if not self.MERGE or not self.has_sibilings(webapp, context_copy):
+                context_copy['fpm_path'] = settings.WEBAPPS_PHPFPM_POOL_PATH % context_copy
+                self.append("rm -f %(fpm_path)s" % context_copy)
+    
+    def delete_fcgid(self, webapp, context, preserve=False):
+        """ delete all pools in order to efectively support changing php-fcgid version """
+        context_copy = dict(context)
+        for php_version, verbose in settings.WEBAPPS_PHP_VERSIONS:
+            if preserve and php_version == context['php_version']:
+                continue
+            php_version_number = utils.extract_version_number(php_version)
+            context_copy['php_version_number'] = php_version_number
+            if not self.MERGE or not self.has_sibilings(webapp, context_copy):
+                context_copy.update({
+                    'wrapper_path': settings.WEBAPPS_FCGID_WRAPPER_PATH % context_copy,
+                    'cmd_options_path': settings.WEBAPPS_FCGID_CMD_OPTIONS_PATH % context_copy,
+                })
+                self.append("rm -f %(wrapper_path)s" % context_copy)
+                self.append("rm -f %(cmd_options_path)s" % context_copy)
     
     def prepare(self):
         super(PHPBackend, self).prepare()
@@ -237,7 +263,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
             return ' \\\n    '.join(cmd_options)
     
     def update_fcgid_context(self, webapp, context):
-        wrapper_path = webapp.type_instance.FCGID_WRAPPER_PATH % context
+        wrapper_path = settings.WEBAPPS_FCGID_WRAPPER_PATH % context
         context.update({
             'wrapper': self.get_fcgid_wrapper(webapp, context),
             'wrapper_path': wrapper_path,

orchestra/contrib/webapps/filters.py

@@ -1,6 +1,8 @@
 from django.contrib.admin import SimpleListFilter
 from django.utils.translation import ugettext_lazy as _
 
+from . import settings
+
 
 class HasWebsiteListFilter(SimpleListFilter):
     title = _("Has website")
@@ -20,3 +22,15 @@ class HasWebsiteListFilter(SimpleListFilter):
         return queryset
 
 
+class PHPVersionListFilter(SimpleListFilter):
+    title = _("PHP version")
+    parameter_name = 'php_version'
+    
+    def lookups(self, request, model_admin):
+        return settings.WEBAPPS_PHP_VERSIONS
+    
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value:
+            return queryset.filter(data__contains='"php_version":"%s"' % value)
+        return queryset

orchestra/contrib/webapps/options.py

@@ -159,6 +159,12 @@ class PHPExtension(PHPAppOption):
     regex = r'^[^ ]+$'
 
 
+class PHPIncludePath(PHPAppOption):
+    name = 'include_path'
+    verbose_name = _("Include path")
+    regex = r'^[^ ]+$'
+
+
 class PHPMagicQuotesGPC(PHPAppOption):
     name = 'magic_quotes_gpc'
     verbose_name = _("Magic quotes GPC")

orchestra/contrib/webapps/settings.py

@@ -30,7 +30,7 @@ WEBAPPS_FPM_DEFAULT_MAX_CHILDREN = Setting('WEBAPPS_FPM_DEFAULT_MAX_CHILDREN',
 
 
 WEBAPPS_PHPFPM_POOL_PATH = Setting('WEBAPPS_PHPFPM_POOL_PATH',
-    '/etc/php5/fpm/pool.d/%(user)s-%(app_name)s.conf',
+    '/etc/php%(php_version_number)s/fpm/pool.d/%(user)s-%(app_name)s.conf',
     help_text="Available fromat names: <tt>%s</tt>" % ', '.join(_php_names),
     validators=[Setting.string_format_validator(_php_names)],
 )
@@ -84,6 +84,8 @@ WEBAPPS_TYPES = Setting('WEBAPPS_TYPES', (
 
 
 WEBAPPS_PHP_VERSIONS = Setting('WEBAPPS_PHP_VERSIONS', (
+        ('5.6-fpm', 'PHP 5.6 FPM'),
+        ('5.6-cgi', 'PHP 5.6 FCGID'),
         ('5.4-fpm', 'PHP 5.4 FPM'),
         ('5.4-cgi', 'PHP 5.4 FCGID'),
         ('5.3-cgi', 'PHP 5.3 FCGID'),
@@ -96,7 +98,7 @@ WEBAPPS_PHP_VERSIONS = Setting('WEBAPPS_PHP_VERSIONS', (
 
 
 WEBAPPS_DEFAULT_PHP_VERSION = Setting('WEBAPPS_DEFAULT_PHP_VERSION',
-    '5.4-cgi',
+    '5.6-fpm',
     choices=WEBAPPS_PHP_VERSIONS
 )
 
@@ -223,6 +225,7 @@ WEBAPPS_ENABLED_OPTIONS = Setting('WEBAPPS_ENABLED_OPTIONS', (
         'orchestra.contrib.webapps.options.PHPDefaultSocketTimeout',
         'orchestra.contrib.webapps.options.PHPDisplayErrors',
         'orchestra.contrib.webapps.options.PHPExtension',
+        'orchestra.contrib.webapps.options.PHPIncludePath',
         'orchestra.contrib.webapps.options.PHPMagicQuotesGPC',
         'orchestra.contrib.webapps.options.PHPMagicQuotesRuntime',
         'orchestra.contrib.webapps.options.PHPMaginQuotesSybase',

orchestra/contrib/webapps/types/php.py

@@ -1,5 +1,4 @@
 import os
-import re
 from collections import OrderedDict
 
 from django import forms
@@ -9,7 +8,7 @@ from rest_framework import serializers
 from orchestra.plugins.forms import PluginDataForm
 from orchestra.utils.functional import cached
 
-from .. import settings
+from .. import settings, utils
 from ..options import AppOption
 
 from . import AppType
@@ -146,9 +145,5 @@ class PHPApp(AppType):
     
     def get_php_version_number(self):
         php_version = self.get_php_version()
-        number = re.findall(r'[0-9]+\.?[0-9]?', php_version)
+        return utils.extract_version_number(php_version)
-        if not number:
+
-            raise ValueError("No version number matches for '%s'" % php_version)
-        if len(number) > 1:
-            raise ValueError("Multiple version number matches for '%s'" % php_version)
-        return number[0]

orchestra/management/commands/setupnginx.py

@@ -152,7 +152,7 @@ class Command(BaseCommand):
             'project_dir': paths.get_project_dir(),
             'site_dir': paths.get_site_dir(),
             'static_root': settings.STATIC_ROOT,
-            'static_url': (settings.STATIC_URL or '/static').rstrip('/')
+            'static_url': (settings.STATIC_URL or '/static').rstrip('/'),
             'user': user,
             'group': options.get('group') or user,
             'home': expanduser("~%s" % options.get('user')),