Fixes on billing

TODO.md

@@ -13,7 +13,10 @@
 
 * backend logs with hal logo
 
-* LAST version of this shit http://wkhtmltopdf.org/downloads.h otml
+# LAST version of this shit http://wkhtmltopdf.org/downloads.h otml
+#apt-get install xfonts-75dpi
+#wget http://downloads.sourceforge.net/wkhtmltopdf/wkhtmltox-0.12.2.1_linux-jessie-amd64.deb
+#dpkg -i wkhtmltox-0.12.2.1_linux-jessie-amd64.deb 
 
 * help_text on readonly_fields specialy Bill.state. (eg. A bill is in OPEN state when bla bla )
 
@@ -363,3 +366,6 @@ pip3 install https://github.com/fantix/gevent/archive/master.zip
 # SIgnal handler for notify workers to reload stuff, like resource sync: https://docs.python.org/2/library/signal.html
 
 # INVOICE fucking Id based on order ID or what?
+
+# user order_id as bill line id 
+# BUG Delete related services also deletes account!

orchestra/bin/orchestra-admin

@@ -123,6 +123,7 @@ function install_requirements () {
     ORCHESTRA_PATH=$(get_orchestra_dir) || true
     
     # lxml: libxml2-dev, libxslt1-dev, zlib1g-dev
+    # wkhtmltopdf: xfonts-75dpi, xvfb
     APT="python3 \
          python3-pip \
          python3-dev \
@@ -131,12 +132,13 @@ function install_requirements () {
          zlib1g-dev \
          bind9utils \
          wkhtmltopdf \
+         xfonts-75dpi \
          xvfb \
          ca-certificates \
          gettext \
          libcrack2-dev"
     
-    # cracklib and lxml are excluded on the requirements because they are hard to build
+    # cracklib and lxml are excluded on the requirements.txt because they need unconvinient system dependencies
     PIP="$(wget https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt -q -O - | tr '\n' ' ') \
         cracklib \
         lxml==3.3.5"
@@ -153,12 +155,17 @@ function install_requirements () {
             coverage \
             flake8 \
             django-debug-toolbar==1.3.0 \
-            https://github.com/django-nose/django-nose/archive/master.zip \
+            django-nose==1.4 \
             sqlparse \
             pyinotify \
             PyMySQL"
     fi
     
+    # Install a more recent version of wkhtmltopdf (0.12.2) (PDF page number support)
+    wkhtmltox=$(mktemp)
+    wget http://downloads.sourceforge.net/wkhtmltopdf/wkhtmltox-0.12.2.1_linux-jessie-amd64.deb -O ${wkhtmltox}
+    dpkg -i ${wkhtmltox}
+    
     # Make sure locales are in place before installing postgres
     if [[ $({ perl --help > /dev/null; } 2>&1|grep 'locale failed') ]]; then
         run sed -i "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen

orchestra/contrib/bills/templates/bills/microspective.css

@@ -4,7 +4,7 @@ body {
     margin: 40 auto !important;
 /*    margin-bottom: 30 !important;*/
     float: none !important;
-    font-family: Arial, 'Liberation Sans', 'DejaVu Sans', sans-serif;
+    font-family: sans;
 }
 
 a {

orchestra/contrib/domains/forms.py

@@ -35,8 +35,7 @@ class BatchDomainCreationAdminForm(forms.ModelForm):
         if not cleaned_data['account']:
             account = None
             for name in [cleaned_data['name']] + self.extra_names:
-                domain = Domain(name=name)
+                parent = Domain.get_parent_domain(name)
-                parent = domain.get_parent()
                 if not parent:
                     # Fake an account to make django validation happy
                     account_model = self.fields['account']._queryset.model

orchestra/contrib/mailer/backends.py

@@ -1,3 +1,4 @@
+from django.conf import settings as djsettings
 from django.core.mail.backends.base import BaseEmailBackend
 
 from .models import Message
@@ -20,7 +21,7 @@ class EmailBackend(BaseEmailBackend):
                 message = Message.objects.create(
                     priority=priority,
                     to_address=to_email,
-                    from_address=message.from_email,
+                    from_address=getattr(message, 'from_email', djsettings.DEFAULT_FROM_EMAIL),
                     subject=message.subject,
                     content=content,
                 )

orchestra/contrib/orders/filters.py

@@ -38,35 +38,48 @@ class BilledOrderListFilter(SimpleListFilter):
         return (
             ('yes', _("Billed")),
             ('no', _("Not billed")),
+            ('pending', _("Pending (re-evaluate metric)")),
+            ('not_pending', _("Not pending (re-evaluate metric)")),
         )
     
+    def get_pending_metric_pks(self, queryset):
+        mindelta = timedelta(days=2) # TODO
+        metric_pks = []
+        prefetch_valid_metrics = Prefetch('metrics', to_attr='valid_metrics',
+            queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'),
+                created_on__lte=(F('updated_on')-mindelta))
+        )
+        prefetch_billed_metric = Prefetch('metrics', to_attr='billed_metric',
+            queryset=MetricStorage.objects.filter(order__billed_on__isnull=False,
+                created_on__lte=F('order__billed_on'), updated_on__gt=F('order__billed_on'))
+        )
+        metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True)
+        for order in metric_queryset.prefetch_related(prefetch_valid_metrics, prefetch_billed_metric):
+            if len(order.billed_metric) != 1:
+                raise ValueError("Data inconsistency #metrics %i != 1." % len(order.billed_metric))
+            billed_metric = order.billed_metric[0].value
+            for metric in order.valid_metrics:
+                if metric.created_on <= order.billed_on:
+                    raise ValueError("This value should already be filtered on the prefetch query.")
+                if metric.value > billed_metric:
+                    metric_pks.append(order.pk)
+                    break
+        return metric_pks
+    
     def queryset(self, request, queryset):
         if self.value() == 'yes':
             return queryset.filter(billed_until__isnull=False, billed_until__gte=timezone.now())
         elif self.value() == 'no':
-            mindelta = timedelta(days=2) # TODO
+            return queryset.exclude(billed_until__isnull=False, billed_until__gte=timezone.now())
-            metric_pks = []
+        elif self.value() == 'pending':
-            prefetch_valid_metrics = Prefetch('metrics', to_attr='valid_metrics',
-                queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'),
-                    created_on__lte=(F('updated_on')-mindelta))
-            )
-            prefetch_billed_metric = Prefetch('metrics', to_attr='billed_metric',
-                queryset=MetricStorage.objects.filter(order__billed_on__isnull=False,
-                    created_on__lte=F('order__billed_on'), updated_on__gt=F('order__billed_on'))
-            )
-            metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True)
-            for order in metric_queryset.prefetch_related(prefetch_valid_metrics, prefetch_billed_metric):
-                if len(order.billed_metric) != 1:
-                    raise ValueError("Data inconsistency.")
-                billed_metric = order.billed_metric[0].value
-                for metric in order.valid_metrics:
-                    if metric.created_on <= order.billed_on:
-                        raise ValueError("This value should already be filtered on the prefetch query.")
-                    if metric.value > billed_metric:
-                        metric_pks.append(order.pk)
-                        break
             return queryset.filter(
-                Q(pk__in=metric_pks) | Q(
+                Q(pk__in=self.get_pending_metric_pks(queryset)) | Q(
+                    Q(billed_until__isnull=True) | Q(billed_until__lt=timezone.now())
+                )
+            )
+        elif self.value() == 'not_pending':
+            return queryset.exclude(
+                Q(pk__in=self.get_pending_metric_pks(queryset)) | Q(
                     Q(billed_until__isnull=True) | Q(billed_until__lt=timezone.now())
                 )
             )

orchestra/contrib/orders/models.py

@@ -59,15 +59,15 @@ class OrderQuerySet(models.QuerySet):
     
     def get_related(self, **options):
         """ returns related orders that could have a pricing effect """
+        # TODO for performance reasons get missing from queryset:
+        # TODO optimize this shit, don't get related if all objects are here
         Service = apps.get_model(settings.ORDERS_SERVICE_MODEL)
         conflictive = self.filter(service__metric='')
-        conflictive = conflictive.exclude(service__billing_period=Service.NEVER)
+        conflictive = conflictive.exclude(service__billing_period=Service.NEVER).exclude(service__rates__isnull=True)
-        conflictive = conflictive.select_related('service').group_by('account_id', 'service')
+        conflictive = conflictive.select_related('service').distinct().group_by('account_id', 'service')
         qs = Q()
         for account_id, services in conflictive.items():
             for service, orders in services.items():
-                if not service.rates.exists():
-                    continue
                 ini = datetime.date.max
                 end = datetime.date.min
                 bp = None
@@ -265,9 +265,15 @@ class MetricStorage(models.Model):
         except cls.DoesNotExist:
             cls.objects.create(order=order, value=value, updated_on=now)
         else:
-            error = decimal.Decimal(str(settings.ORDERS_METRIC_ERROR))
+            # Metric storage has per-day granularity (last value of the day is what counts)
-            if value > last.value+error or value < last.value-error:
+            if last.created_on == now.date():
-                cls.objects.create(order=order, value=value, updated_on=now)
+                last.value = value
-            else:
                 last.updated_on = now
-                last.save(update_fields=['updated_on'])
+                last.save()
+            else:
+                error = decimal.Decimal(str(settings.ORDERS_METRIC_ERROR))
+                if value > last.value+error or value < last.value-error:
+                    cls.objects.create(order=order, value=value, updated_on=now)
+                else:
+                    last.updated_on = now
+                    last.save(update_fields=['updated_on'])

orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html

@@ -35,7 +35,7 @@
             <h2><a href="{% url 'admin:accounts_account_change' account.pk %}">{{ account }}</a><span style="float:right">{{ total | floatformat:"-2" }} &euro;</span></h2>
             <table>
             <thead>
-                <tr><th style="width:30%;">Description</th> <th style="width:30%;">Period</th> <th style="width:10%;">Quantity</th> <th style="width:10%;">Price</th></tr>
+                <tr><th style="width:30%;">Description</th> <th style="width:30%;">Period</th> <th style="width:10%;">Size&times;Quantity</th> <th style="width:10%;">Price</th></tr>
             </thead>
             <tbody>
             {% for line in lines %}
@@ -47,7 +47,7 @@
                     {% endfor %}
                 </td>
                 <td>{{ line.ini | date }} to {{ line.end | date }}</td>
-                <td>{{ line.size | floatformat:"-2" }}</td>
+                <td>{{ line.size | floatformat:"-2" }}&times;{{ line.metric | floatformat:"-2"}}</td>
                 <td>
                     &nbsp;{{ line.subtotal | floatformat:"-2" }} &euro;
                     {% for discount in line.discounts %}<br>{{ discount.total | floatformat:"-2" }} &euro;{% endfor %}

orchestra/utils/html.py

@@ -1,12 +1,19 @@
+import textwrap
+
 from orchestra.utils.sys import run
 
 
 def html_to_pdf(html):
     """ converts HTL to PDF using wkhtmltopdf """
-    return run(
+    return run(textwrap.dedent("""\
-        'PATH=$PATH:/usr/local/bin/\n'
+        PATH=$PATH:/usr/local/bin/
-        'xvfb-run -a -s "-screen 0 640x4800x16" '
+        xvfb-run -a -s "-screen 0 2480x3508x16" wkhtmltopdf -q \\
-            'wkhtmltopdf -q --footer-center "Page [page] of [topage]" '
+            --use-xserver \\
-            '   --footer-font-size 9 --margin-bottom 20 --margin-top 20 - -',
+            --footer-center "Page [page] of [topage]" \\
+            --footer-font-name sans \\
+            --footer-font-size 7 \\
+            --footer-spacing 7 \\
+            --margin-bottom 22 \\
+            --margin-top 20 - - """),
         stdin=html.encode('utf-8')
     ).stdout