pedro/django-orchestra-test
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Added metricstorage cleanup periodic task

Browse source
This commit is contained in:
Marc Aymerich 2015-09-10 10:34:07 +00:00
parent 332ad49b64
commit 2beed2677a
7 changed files with 82 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
TODO.md
View file

@ -182,7 +182,6 @@ ugettext("Description")
* saas validate_creation generic approach, for all backends. standard output * saas validate_creation generic approach, for all backends. standard output
* periodic task to cleanup metricstorage
# create orchestrate databases.Database pk=1 -n --dry-run | --noinput --action save (default)|delete --backend name (limit to this backend) --help # create orchestrate databases.Database pk=1 -n --dry-run | --noinput --action save (default)|delete --backend name (limit to this backend) --help
* postupgradeorchestra send signals in order to hook custom stuff * postupgradeorchestra send signals in order to hook custom stuff
@ -394,6 +393,3 @@ Case
# Don't enforce one contact per account? remove account.email in favour of contacts? # Don't enforce one contact per account? remove account.email in favour of contacts?
# Mailer: mark as sent # Mailer: mark as sent
# Pending filter filter out orders zero metric from pending

2
orchestra/contrib/domains/models.py
View file

@ -33,7 +33,7 @@ class Domain(models.Model):
account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), blank=True, account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), blank=True,
related_name='domains', help_text=_("Automatically selected for subdomains.")) related_name='domains', help_text=_("Automatically selected for subdomains."))
top = models.ForeignKey('domains.Domain', null=True, related_name='subdomain_set', top = models.ForeignKey('domains.Domain', null=True, related_name='subdomain_set',
editable=False) editable=False, verbose_name=_("top domain"))
serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, editable=False, serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, editable=False,
help_text=_("A revision number that changes whenever this domain is updated.")) help_text=_("A revision number that changes whenever this domain is updated."))
refresh = models.CharField(_("refresh"), max_length=16, blank=True, refresh = models.CharField(_("refresh"), max_length=16, blank=True,

6
orchestra/contrib/orders/settings.py
View file

@ -36,3 +36,9 @@ ORDERS_METRIC_ERROR = Setting('ORDERS_METRIC_ERROR',
help_text=("Only account for significative changes.<br>" help_text=("Only account for significative changes.<br>"
"metric_storage new value: <tt>lastvalue*(1+threshold) > currentvalue or lastvalue*threshold < currentvalue</tt>."), "metric_storage new value: <tt>lastvalue*(1+threshold) > currentvalue or lastvalue*threshold < currentvalue</tt>."),
) )
ORDERS_BILLED_METRIC_CLEANUP_DAYS = Setting('ORDERS_BILLED_METRIC_CLEANUP_DAYS',
40,
help_text=("Number of days after a billed stored metric is deleted."),
)

50
orchestra/contrib/orders/tasks.py Normal file
View file

@ -0,0 +1,50 @@
import datetime
from celery.task.schedules import crontab
from django.apps import apps
from orchestra.contrib.tasks import periodic_task
from . import settings
@periodic_task(run_every=crontab(hour=4, minute=30), name='orders.cleanup_metrics')
def cleanup_metrics():
from .models import MetricStorage, Order
Service = apps.get_model(settings.ORDERS_SERVICE_MODEL)
# General cleaning: order.billed_on-delta
general = 0
delta = datetime.timedelta(days=settings.ORDERS_BILLED_METRIC_CLEANUP_DAYS)
for order in Order.objects.filter(billed_on__isnull=False):
epoch = order.billed_on-delta
try:
latest = order.metrics.filter(updated_on__lt=epoch).latest('updated_on')
except MetricStorage.DoesNotExist:
pass
else:
general += order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).count()
order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).only('id').delete()
# Reduce monthly metrics to latest
monthly = 0
monthly_services = Service.objects.exclude(metric='').filter(
billing_period=Service.MONTHLY, pricing_period=Service.BILLING_PERIOD
)
for service in monthly_services:
for order in Order.objects.filter(service=service):
dates = order.metrics.values_list('created_on', flat=True)
months = set((date.year, date.month) for date in dates)
for year, month in months:
metrics = order.metrics.filter(
created_on__year=year, created_on__month=month,
updated_on__year=year, updated_on__month=month)
try:
latest = metrics.latest('updated_on')
except MetricStorage.DoesNotExist:
pass
else:
monthly += metrics.exclude(pk=latest.pk).count()
metrics.exclude(pk=latest.pk).only('id').delete()
return (general, monthly)

30
orchestra/contrib/saas/backends/wordpressmu.py
View file

@ -40,18 +40,18 @@ class WordpressMuBackend(ServiceController):
errors = re.findall(r'<body id="error-page">\n\t<p>(.*)</p></body>', response.content.decode('utf8')) errors = re.findall(r'<body id="error-page">\n\t<p>(.*)</p></body>', response.content.decode('utf8'))
raise RuntimeError(errors[0] if errors else 'Unknown %i error' % response.status_code) raise RuntimeError(errors[0] if errors else 'Unknown %i error' % response.status_code)
def get_id(self, session, webapp): def get_id(self, session, saas):
search = self.get_base_url() search = self.get_base_url()
search += '/wp-admin/network/sites.php?s=%s&action=blogs' % webapp.name search += '/wp-admin/network/sites.php?s=%s&action=blogs' % saas.name
regex = re.compile( regex = re.compile(
'<a href="http://[\.\-\w]+/wp-admin/network/site-info\.php\?id=([0-9]+)"\s+' '<a href="http://[\.\-\w]+/wp-admin/network/site-info\.php\?id=([0-9]+)"\s+'
'class="edit">%s</a>' % webapp.name 'class="edit">%s</a>' % saas.name
) )
content = session.get(search).content.decode('utf8') content = session.get(search).content.decode('utf8')
# Get id # Get id
ids = regex.search(content) ids = regex.search(content)
if not ids: if not ids:
raise RuntimeError("Blog '%s' not found" % webapp.name) raise RuntimeError("Blog '%s' not found" % saas.name)
ids = ids.groups() ids = ids.groups()
if len(ids) > 1: if len(ids) > 1:
raise ValueError("Multiple matches") raise ValueError("Multiple matches")
@ -60,13 +60,13 @@ class WordpressMuBackend(ServiceController):
wpnonce = re.search(r'_wpnonce=([^"]*)"', wpnonce).groups()[0] wpnonce = re.search(r'_wpnonce=([^"]*)"', wpnonce).groups()[0]
return int(ids[0]), wpnonce return int(ids[0]), wpnonce
def create_blog(self, webapp, server): def create_blog(self, saas, server):
session = requests.Session() session = requests.Session()
self.login(session) self.login(session)
# Check if blog already exists # Check if blog already exists
try: try:
self.get_id(session, webapp) self.get_id(session, saas)
except RuntimeError: except RuntimeError:
url = self.get_base_url() url = self.get_base_url()
url += '/wp-admin/network/site-new.php' url += '/wp-admin/network/site-new.php'
@ -77,9 +77,9 @@ class WordpressMuBackend(ServiceController):
url += '?action=add-site' url += '?action=add-site'
data = { data = {
'blog[domain]': webapp.name, 'blog[domain]': saas.name,
'blog[title]': webapp.name, 'blog[title]': saas.name,
'blog[email]': webapp.account.email, 'blog[email]': saas.account.email,
'_wpnonce_add-blog': wpnonce, '_wpnonce_add-blog': wpnonce,
} }
@ -87,12 +87,12 @@ class WordpressMuBackend(ServiceController):
response = session.post(url, data=data) response = session.post(url, data=data)
self.validate_response(response) self.validate_response(response)
def delete_blog(self, webapp, server): def delete_blog(self, saas, server):
session = requests.Session() session = requests.Session()
self.login(session) self.login(session)
try: try:
id, wpnonce = self.get_id(session, webapp) id, wpnonce = self.get_id(session, saas)
except RuntimeError: except RuntimeError:
pass pass
else: else:
@ -114,8 +114,8 @@ class WordpressMuBackend(ServiceController):
response = session.post(delete, data=data) response = session.post(delete, data=data)
self.validate_response(response) self.validate_response(response)
def save(self, webapp): def save(self, saas):
self.append(self.create_blog, webapp) self.append(self.create_blog, saas)
def delete(self, webapp): def delete(self, saas):
self.append(self.delete_blog, webapp) self.append(self.delete_blog, saas)

2
orchestra/contrib/saas/settings.py
View file

@ -23,7 +23,7 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES',
) )
SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESSMU_ADMIN_PASSWORD', SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESS_ADMIN_PASSWORD',
'secret' 'secret'
) )

14
orchestra/contrib/services/handlers.py
View file

@ -25,6 +25,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
""" """
_PLAN = 'plan' _PLAN = 'plan'
_COMPENSATION = 'compensation' _COMPENSATION = 'compensation'
_PREPAY = 'prepay'
model = None model = None
@ -275,14 +276,15 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
'metric': metric, 'metric': metric,
'discounts': [], 'discounts': [],
}) })
if subtotal > price: if subtotal > price:
plan_discount = price-subtotal plan_discount = price-subtotal
self.generate_discount(line, self._PLAN, plan_discount) self.generate_discount(line, self._PLAN, plan_discount)
subtotal += plan_discount subtotal += plan_discount
for dtype, dprice in discounts: for dtype, dprice in discounts:
subtotal += dprice subtotal += dprice
# Prevent compensations to refund money # Prevent compensations/prepays to refund money
if dtype == self._COMPENSATION and subtotal < 0: if dtype in (self._COMPENSATION, self._PREPAY) and subtotal < 0:
dprice -= subtotal dprice -= subtotal
if dprice: if dprice:
self.generate_discount(line, dtype, dprice) self.generate_discount(line, dtype, dprice)
@ -527,7 +529,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
if discount > 0: if discount > 0:
price -= discount price -= discount
discounts = ( discounts = (
('prepay', -discount), (self._PREPAY, -discount),
) )
# Don't overdload bills with lots of lines # Don't overdload bills with lots of lines
if price > 0: if price > 0:
@ -535,6 +537,8 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
if prepay_discount < 0: if prepay_discount < 0:
# User has prepaid less than the actual consumption # User has prepaid less than the actual consumption
for order, price, cini, cend, metric, discounts in recharges: for order, price, cini, cend, metric, discounts in recharges:
if discounts:
price -= discounts[0][1]
line = self.generate_line(order, price, cini, cend, metric=metric, line = self.generate_line(order, price, cini, cend, metric=metric,
computed=True, discounts=discounts) computed=True, discounts=discounts)
lines.append(line) lines.append(line)
@ -561,7 +565,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
# price -= discount # price -= discount
# prepay_discount -= discount # prepay_discount -= discount
# discounts = ( # discounts = (
# ('prepay', -discount), # (self._PREPAY', -discount),
# ) # )
if metric > 0: if metric > 0:
line = self.generate_line(order, price, cini, cend, metric=metric, line = self.generate_line(order, price, cini, cend, metric=metric,
@ -580,7 +584,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
# price -= discount # price -= discount
# prepay_discount -= discount # prepay_discount -= discount
# discounts = ( # discounts = (
# ('prepay', -discount), # (self._PREPAY, -discount),
# ) # )
if metric > 0: if metric > 0:
line = self.generate_line(order, price, cini, cend, metric=metric, line = self.generate_line(order, price, cini, cend, metric=metric,