Merge branch 'master' into 30-application-security-gateway

# Conflicts:
#	passbook/core/policies.py
#	passbook/core/settings.py

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.23-beta
+current_version = 0.1.24-beta
 tag = True
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)\-(?P<release>.*)

.gitlab-ci.yml

@@ -12,6 +12,7 @@ stages:
 image: python:3.6
 services:
     - postgres:latest
+    - redis:latest
 
 variables:
     POSTGRES_DB: passbook
@@ -54,7 +55,7 @@ package-docker:
     before_script:
         - echo "{\"auths\":{\"docker.$NEXUS_URL\":{\"auth\":\"$NEXUS_AUTH\"}}}" > /kaniko/.docker/config.json
     script:
-        - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.23-beta
+        - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination docker.pkg.beryju.org/passbook:latest --destination docker.pkg.beryju.org/passbook:0.1.24-beta
     stage: build
     only:
         - tags

client-packages/allauth/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup
 
 setup(
     name='django-allauth-passbook',
-    version='0.1.23-beta',
+    version='0.1.24-beta',
     description='passbook support for django-allauth',
     # long_description='\n'.join(read_simple('docs/index.md')[2:]),
     long_description_content_type='text/markdown',

client-packages/sentry-auth-passbook/setup.py

@@ -18,7 +18,7 @@ tests_require = [
 
 setup(
     name='sentry-auth-passbook',
-    version='0.1.23-beta',
+    version='0.1.24-beta',
     author='BeryJu.org',
     author_email='support@beryju.org',
     url='https://passbook.beryju.org',

debian/changelog

@@ -1,3 +1,11 @@
+passbook (0.1.24) stable; urgency=medium
+
+  * bump version: 0.1.22-beta -> 0.1.23-beta
+  * add modal for OAuth Providers showing the URLs
+  * remove user field from form. Closes #32
+
+ -- Jens Langhammer <jens.langhammer@beryju.org>  Wed, 20 Mar 2019 21:59:21 +0000
+
 passbook (0.1.23) stable; urgency=medium
 
   * add support for OpenID-Connect Discovery

debian/control

@@ -8,7 +8,7 @@ Standards-Version: 3.9.6
 
 Package: passbook
 Architecture: all
-Recommends: mysql-server, rabbitmq-server
+Recommends: mysql-server, rabbitmq-server, redis-server
 Pre-Depends: adduser, libldap2-dev, libsasl2-dev
 Depends: python3 (>= 3.5) | python3.6 | python3.7, python3-pip, dbconfig-pgsql | dbconfig-no-thanks, ${misc:Depends}
 Description: Authentication Provider/Proxy supporting protocols like SAML, OAuth, LDAP and more.

debian/etc/passbook/config.yml

@@ -11,6 +11,8 @@ debug: false
 secure_proxy_header:
   HTTP_X_FORWARDED_PROTO: https
 rabbitmq: guest:guest@localhost/passbook
+redis: localhost/0
+
 # Error reporting, sends stacktrace to sentry.services.beryju.org
 error_report_enabled: true
 

helm/passbook/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
-appVersion: "0.1.23-beta"
+appVersion: "0.1.24-beta"
 description: A Helm chart for passbook.
 name: passbook
-version: "0.1.23-beta"
+version: "0.1.24-beta"
 icon: https://passbook.beryju.org/images/logo.png

helm/passbook/requirements.lock

@@ -5,5 +5,8 @@ dependencies:
 - name: postgresql
   repository: https://kubernetes-charts.storage.googleapis.com/
   version: 3.10.1
-digest: sha256:c36e054785f7d706d7d3f525eb1b167dbc89b42f84da7fc167a18bbb6542c999
+- name: redis
-generated: 2019-03-11T20:36:35.125079+01:00
+  repository: https://kubernetes-charts.storage.googleapis.com/
+  version: 5.1.0
+digest: sha256:8bf68bc928a2e3c0f05139635be05fa0840554c7bde4cecd624fac78fb5fa5a3
+generated: 2019-03-21T11:06:51.553379+01:00

helm/passbook/requirements.yaml

@@ -5,3 +5,6 @@ dependencies:
 - name: postgresql
   version: 3.10.1
   repository: https://kubernetes-charts.storage.googleapis.com/
+- name: redis
+  version: 5.1.0
+  repository: https://kubernetes-charts.storage.googleapis.com/

helm/passbook/templates/passbook-configmap.yaml

@@ -37,6 +37,7 @@ data:
     secure_proxy_header:
       HTTP_X_FORWARDED_PROTO: https
     rabbitmq: "user:{{ .Values.rabbitmq.rabbitmq.password }}@{{ .Release.Name }}-rabbitmq"
+    redis: ":{{ .Values.redis.password }}@{{ .Release.Name }}-redis-master/0"
     # Error reporting, sends stacktrace to sentry.services.beryju.org
     error_report_enabled: {{ .Values.config.error_reporting }}
 

helm/passbook/values.yaml

@@ -5,7 +5,7 @@
 replicaCount: 1
 
 image:
-  tag: 0.1.23-beta
+  tag: 0.1.24-beta
 
 nameOverride: ""
 

passbook/__init__.py

@@ -1,2 +1,2 @@
 """passbook"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/admin/__init__.py

@@ -1,2 +1,2 @@
 """passbook admin"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/admin/templates/administration/provider/list.html

@@ -57,6 +57,10 @@
                     <a class="btn btn-default btn-sm"
                         href="{{ href }}?back={{ request.get_full_path }}">{% trans name %}</a>
                     {% endfor %}
+                    {% get_htmls provider as htmls %}
+                    {% for html in htmls %}
+                    {{ html|safe }}
+                    {% endfor %}
                 </td>
             </tr>
             {% endfor %}

passbook/admin/templatetags/admin_reflection.py

@@ -5,6 +5,8 @@ from logging import getLogger
 from django import template
 from django.db.models import Model
 
+from passbook.lib.utils.template import render_to_string
+
 register = template.Library()
 LOGGER = getLogger(__name__)
 
@@ -29,3 +31,24 @@ def get_links(model_instance):
         pass
 
     return links
+
+
+@register.simple_tag(takes_context=True)
+def get_htmls(context, model_instance):
+    """Find all html_ methods on an object instance, run them and return as dict"""
+    prefix = 'html_'
+    htmls = []
+
+    if not isinstance(model_instance, Model):
+        LOGGER.warning("Model %s is not instance of Model", model_instance)
+        return htmls
+
+    try:
+        for name, method in inspect.getmembers(model_instance, predicate=inspect.ismethod):
+            if name.startswith(prefix):
+                template, _context = method(context.get('request'))
+                htmls.append(render_to_string(template, _context))
+    except NotImplementedError:
+        pass
+
+    return htmls

passbook/api/__init__.py

@@ -1,2 +1,2 @@
 """passbook api"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/audit/__init__.py

@@ -1,2 +1,2 @@
 """passbook audit Header"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/captcha_factor/__init__.py

@@ -1,2 +1,2 @@
 """passbook captcha_factor Header"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/core/__init__.py

@@ -1,2 +1,2 @@
 """passbook core"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/core/policies.py

@@ -3,7 +3,11 @@ from logging import getLogger
 
 from amqp.exceptions import UnexpectedFrame
 from celery import group
+<<<<<<< HEAD
 from celery.exceptions import TimeoutError as CeleryTimeoutError
+=======
+from django.core.cache import cache
+>>>>>>> master
 from ipware import get_client_ip
 
 from passbook.core.celery import CELERY_APP
@@ -11,6 +15,9 @@ from passbook.core.models import Policy, User
 
 LOGGER = getLogger(__name__)
 
+def _cache_key(policy, user):
+    return "%s#%s" % (policy.uuid, user.pk)
+
 @CELERY_APP.task()
 def _policy_engine_task(user_pk, policy_pk, **kwargs):
     """Task wrapper to run policy checking"""
@@ -31,62 +38,81 @@ def _policy_engine_task(user_pk, policy_pk, **kwargs):
     if policy_obj.negate:
         policy_result = not policy_result
     LOGGER.debug("Policy %r#%s got %s", policy_obj.name, policy_obj.pk.hex, policy_result)
+    cache_key = _cache_key(policy_obj, user_obj)
+    cache.set(cache_key, (policy_obj.action, policy_result, message))
+    LOGGER.debug("Cached entry as %s", cache_key)
     return policy_obj.action, policy_result, message
 
 class PolicyEngine:
     """Orchestrate policy checking, launch tasks and return result"""
 
+    __group = None
+    __cached = None
+
     policies = None
-    _group = None
+    __get_timeout = 0
-    _request = None
+    __request = None
-    _user = None
+    __user = None
-    _get_timeout = 0
 
     def __init__(self, policies):
         self.policies = policies
-        self._request = None
+        self.__request = None
-        self._user = None
+        self.__user = None
 
     def for_user(self, user):
         """Check policies for user"""
-        self._user = user
+        self.__user = user
         return self
 
     def with_request(self, request):
         """Set request"""
-        self._request = request
+        self.__request = request
         return self
 
     def build(self):
         """Build task group"""
-        if not self._user:
+        if not self.__user:
             raise ValueError("User not set.")
         signatures = []
+        cached_policies = []
         kwargs = {
-            '__password__': getattr(self._user, '__password__', None),
+            '__password__': getattr(self.__user, '__password__', None),
         }
-        if self._request:
+        if self.__request:
-            kwargs['remote_ip'], _ = get_client_ip(self._request)
+            kwargs['remote_ip'], _ = get_client_ip(self.__request)
             if not kwargs['remote_ip']:
                 kwargs['remote_ip'] = '255.255.255.255'
         for policy in self.policies:
-            signatures.append(_policy_engine_task.signature(
+            cached_policy = cache.get(_cache_key(policy, self.__user), None)
-                args=(self._user.pk, policy.pk.hex),
+            if cached_policy:
-                kwargs=kwargs,
+                LOGGER.debug("Taking result from cache for %s", policy.pk.hex)
-                time_limit=policy.timeout))
+                cached_policies.append(cached_policy)
-            self._get_timeout += policy.timeout
+            else:
-        self._get_timeout += 3
+                LOGGER.debug("Evaluating policy %s", policy.pk.hex)
-        self._get_timeout = (self._get_timeout / len(self.policies)) * 1.5
+                signatures.append(_policy_engine_task.signature(
-        LOGGER.debug("Set total policy timeout to %r", self._get_timeout)
+                    args=(self._user.pk, policy.pk.hex),
-        self._group = group(signatures)()
+                    kwargs=kwargs,
+                    time_limit=policy.timeout))
+                self.__get_timeout += policy.timeout
+        self.__get_timeout += 3
+        self.__get_timeout = (self.__get_timeout / len(self.policies)) * 1.5
+        LOGGER.debug("Set total policy timeout to %r", self.__get_timeout)
+        # If all policies are cached, we have an empty list here.
+        if signatures:
+            self.__group = group(signatures)()
+        self.__cached = cached_policies
         return self
 
     @property
     def result(self):
         """Get policy-checking result"""
         messages = []
+        result = []
         try:
-            group_result = self._group.get(timeout=self._get_timeout)
+            if self.__group:
+                # ValueError can be thrown from _policy_engine_task when user is None
+                result += self.__group.get(timeout=self._get_timeout)
+            result += self.__cached
         except ValueError as exc:
             # ValueError can be thrown from _policy_engine_task when user is None
             return False, [str(exc)]
@@ -94,7 +120,7 @@ class PolicyEngine:
             return False, [str(exc)]
         except CeleryTimeoutError as exc:
             return False, [str(exc)]
-        for policy_action, policy_result, policy_message in group_result:
+        for policy_action, policy_result, policy_message in result:
             passing = (policy_action == Policy.ACTION_ALLOW and policy_result) or \
                       (policy_action == Policy.ACTION_DENY and not policy_result)
             LOGGER.debug('Action=%s, Result=%r => %r', policy_action, policy_result, passing)

passbook/core/requirements.txt

@@ -1,12 +1,13 @@
-django>=2.0
+celery
-django-model-utils
+cherrypy
+colorlog
 django-ipware
+django-model-utils
+django-redis
+django>=2.0
 djangorestframework
+idna<2.8,>=2.5
+markdown
+psycopg2
 PyYAML
 raven
-markdown
-colorlog
-celery
-psycopg2
-idna<2.8,>=2.5
-cherrypy

passbook/core/settings.py

@@ -46,6 +46,8 @@ AUTH_USER_MODEL = 'passbook_core.User'
 CSRF_COOKIE_NAME = 'passbook_csrf'
 SESSION_COOKIE_NAME = 'passbook_session'
 SESSION_COOKIE_DOMAIN = CONFIG.get('primary_domain')
+SESSION_ENGINE = "django.contrib.sessions.backends.cache"
+SESSION_CACHE_ALIAS = "default"
 LANGUAGE_COOKIE_NAME = 'passbook_language'
 
 AUTHENTICATION_BACKENDS = [
@@ -101,6 +103,16 @@ REST_FRAMEWORK = {
     ]
 }
 
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": "redis://%s" % CONFIG.get('redis'),
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        }
+    }
+}
+
 MIDDLEWARE = [
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',

passbook/core/signals.py

@@ -1,10 +1,15 @@
 """passbook core signals"""
+from logging import getLogger
 
+from django.core.cache import cache
 from django.core.signals import Signal
+from django.db.models.signals import post_save
 from django.dispatch import receiver
 
 from passbook.core.exceptions import PasswordPolicyInvalid
 
+LOGGER = getLogger(__name__)
+
 user_signed_up = Signal(providing_args=['request', 'user'])
 invitation_created = Signal(providing_args=['request', 'invitation'])
 invitation_used = Signal(providing_args=['request', 'invitation', 'user'])
@@ -24,3 +29,14 @@ def password_policy_checker(sender, password, **kwargs):
         passing, messages = policy_engine.result
         if not passing:
             raise PasswordPolicyInvalid(*messages)
+
+@receiver(post_save)
+# pylint: disable=unused-argument
+def invalidate_policy_cache(sender, instance, **kwargs):
+    """Invalidate Policy cache when policy is updated"""
+    from passbook.core.models import Policy
+    if isinstance(instance, Policy):
+        LOGGER.debug("Invalidating cache for %s", instance.pk)
+        keys = cache.keys("%s#*" % instance.pk)
+        cache.delete_many(keys)
+        LOGGER.debug("Deleted %d keys", len(keys))

passbook/hibp_policy/__init__.py

@@ -1,2 +1,2 @@
 """passbook hibp_policy"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/ldap/__init__.py

@@ -1,2 +1,2 @@
 """Passbook ldap app Header"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/lib/__init__.py

@@ -1,2 +1,2 @@
 """passbook lib"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/lib/default.yml

@@ -30,6 +30,7 @@ debug: false
 secure_proxy_header:
   HTTP_X_FORWARDED_PROTO: https
 rabbitmq: guest:guest@localhost/passbook
+redis: localhost/0
 # Error reporting, sends stacktrace to sentry.services.beryju.org
 error_report_enabled: true
 secret_key: 9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s

passbook/oauth_client/__init__.py

@@ -1,2 +1,2 @@
 """passbook oauth_client Header"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/oauth_provider/__init__.py

@@ -1,2 +1,2 @@
 """passbook oauth_provider Header"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/oauth_provider/forms.py

@@ -11,5 +11,5 @@ class OAuth2ProviderForm(forms.ModelForm):
     class Meta:
 
         model = OAuth2Provider
-        fields = ['name', 'user', 'redirect_uris', 'client_type',
+        fields = ['name', 'redirect_uris', 'client_type',
                   'authorization_grant_type', 'client_id', 'client_secret', ]

passbook/oauth_provider/models.py

@@ -1,5 +1,6 @@
 """Oauth2 provider product extension"""
 
+from django.shortcuts import reverse
 from django.utils.translation import gettext as _
 from oauth2_provider.models import AbstractApplication
 
@@ -14,6 +15,20 @@ class OAuth2Provider(Provider, AbstractApplication):
     def __str__(self):
         return "OAuth2 Provider %s" % self.name
 
+    def html_setup_urls(self, request):
+        """return template and context modal with URLs for authorize, token, openid-config, etc"""
+        return "oauth2_provider/setup_url_modal.html", {
+            'provider': self,
+            'authorize_url': request.build_absolute_uri(
+                reverse('passbook_oauth_provider:oauth2-authorize')),
+            'token_url': request.build_absolute_uri(
+                reverse('passbook_oauth_provider:token')),
+            'userinfo_url': request.build_absolute_uri(
+                reverse('passbook_api:openid')),
+            'openid_url': request.build_absolute_uri(
+                reverse('passbook_oauth_provider:openid-discovery'))
+        }
+
     class Meta:
 
         verbose_name = _('OAuth2 Provider')

passbook/oauth_provider/templates/oauth2_provider/setup_url_modal.html

@@ -0,0 +1,49 @@
+{% load i18n %}
+
+<button class="btn btn-default btn-sm" data-toggle="modal" data-target="#{{ provider.pk }}">{% trans 'View Setup URLs' %}</button>
+<div class="modal fade" id="{{ provider.pk }}" tabindex="-1" role="dialog" aria-labelledby="{{ provider.pk }}Label" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <button type="button" class="close" data-dismiss="modal" aria-hidden="true" aria-label="Close">
+          <span class="pficon pficon-close"></span>
+        </button>
+        <h4 class="modal-title" id="{{ provider.pk }}Label">{% trans 'Setup URLs' %}</h4>
+      </div>
+      <div class="modal-body">
+        <form class="form-horizontal">
+          <div class="form-group">
+            <label class="col-sm-3 control-label">{% trans 'Authroize URL' %}</label>
+            <div class="col-sm-9">
+              <input type="text"class="form-control" readonly value="{{ authorize_url }}">
+            </div>
+          </div>
+          <div class="form-group">
+            <label class="col-sm-3 control-label">{% trans 'Token URL' %}</label>
+            <div class="col-sm-9">
+              <input type="text" class="form-control" readonly value="{{ token_url }}">
+            </div>
+          </div>
+          <div class="form-group">
+            <label class="col-sm-3 control-label">{% trans 'Userinfo Endpoint' %}</label>
+            <div class="col-sm-9">
+              <input type="text" class="form-control" readonly value="{{ userinfo_url }}">
+            </div>
+          </div>
+        </form>
+        <hr>
+        <form class="form-horizontal">
+          <div class="form-group">
+            <label class="col-sm-3 control-label">{% trans 'OpenID Configuration URL' %}</label>
+            <div class="col-sm-9">
+              <input type="text"class="form-control" readonly value="{{ openid_url }}">
+            </div>
+          </div>
+        </form>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-primary" data-dismiss="modal">{% trans 'Close' %}</button>
+      </div>
+    </div>
+  </div>
+</div>

passbook/otp/__init__.py

@@ -1,2 +1,2 @@
 """passbook otp Header"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/password_expiry_policy/__init__.py

@@ -1,2 +1,2 @@
 """passbook password_expiry"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'

passbook/saml_idp/__init__.py

@@ -1,2 +1,2 @@
 """passbook saml_idp Header"""
-__version__ = '0.1.23-beta'
+__version__ = '0.1.24-beta'