trustchain-oc1-orchestral
/
authentik
Archived
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

config(minor): CONFIG.get -> CONFIG.y

This commit is contained in:
Langhammer, Jens 2019-09-30 18:04:04 +02:00
parent 9cddab8fd5
commit c2c5ff6912
11 changed files with 167 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Pipfile
View File

@ -41,6 +41,7 @@ service_identity = "*"
signxml = "*"
urllib3 = {extras = ["secure"],version = "*"}
websocket_client = "*"
structlog = "*"
[requires]
python_version = "3.7"
@ -57,3 +58,4 @@ unittest-xml-reporting = "*"
autopep8 = "*"
bandit = "*"
twine = "*"
colorama = "*"

18
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "cd82871d9aca8cfd548a6a62856196b2211524f12fbd416dfe5218aad9471e44"
"sha256": "f8694b0ee03f99560e853fd24e9cd7ac987c757cd50249398346e42cdd98cbbb"
},
"pipfile-spec": 6,
"requires": {
@ -777,6 +777,14 @@
],
"version": "==0.3.0"
},
"structlog": {
"hashes": [
"sha256:5feae03167620824d3ae3e8915ea8589fc28d1ad6f3edf3cc90ed7c7cb33fab5",
"sha256:db441b81c65b0f104a7ce5d86c5432be099956b98b8a2c8be0b3fb3a7a0b1536"
],
"index": "pypi",
"version": "==19.1.0"
},
"tempora": {
"hashes": [
"sha256:cb60b1d2b1664104e307f8e5269d7f4acdb077c82e35cd57246ae14a3427d2d6",
@ -950,6 +958,14 @@
],
"version": "==3.0.4"
},
"colorama": {
"hashes": [
"sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
"sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"
],
"index": "pypi",
"version": "==0.4.1"
},
"coverage": {
"hashes": [
"sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6",

2
passbook/core/auth/factor.py
View File

@ -19,7 +19,7 @@ class AuthenticationFactor(TemplateView):
self.authenticator = authenticator
def get_context_data(self, **kwargs):
kwargs['config'] = CONFIG.get('passbook')
kwargs['config'] = CONFIG.y('passbook')
kwargs['is_login'] = True
kwargs['title'] = _('Log in to your account')
kwargs['primary_action'] = _('Log in')

2
passbook/core/management/commands/web.py
View File

@ -17,7 +17,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
"""passbook cherrypy server"""
cherrypy.config.update(CONFIG.get('web'))
cherrypy.config.update(CONFIG.y('web'))
cherrypy.tree.graft(application, '/')
# Mount NullObject to serve static files
cherrypy.tree.mount(None, settings.STATIC_URL, config={

4
passbook/core/views/authentication.py
View File

@ -40,7 +40,7 @@ class LoginView(UserPassesTestMixin, FormView):
return redirect(reverse('passbook_core:overview'))
def get_context_data(self, **kwargs):
kwargs['config'] = CONFIG.get('passbook')
kwargs['config'] = CONFIG.y('passbook')
kwargs['is_login'] = True
kwargs['title'] = _('Log in to your account')
kwargs['primary_action'] = _('Log in')
@ -135,7 +135,7 @@ class SignUpView(UserPassesTestMixin, FormView):
return super().get_initial()
def get_context_data(self, **kwargs):
kwargs['config'] = CONFIG.get('passbook')
kwargs['config'] = CONFIG.y('passbook')
kwargs['is_login'] = True
kwargs['title'] = _('Sign Up')
kwargs['primary_action'] = _('Sign up')

2
passbook/core/views/user.py
View File

@ -66,7 +66,7 @@ class UserChangePasswordView(LoginRequiredMixin, FormView):
return redirect('passbook_core:overview')
def get_context_data(self, **kwargs):
kwargs['config'] = CONFIG.get('passbook')
kwargs['config'] = CONFIG.y('passbook')
kwargs['is_login'] = True
kwargs['title'] = _('Change Password')
kwargs['primary_action'] = _('Change')

2
passbook/ldap/ldap_connector.py
View File

@ -166,7 +166,7 @@ class LDAPConnector:
if not self._source.enabled:
return None
# FIXME: Adapt user_uid
# email = filters.pop(CONFIG.get('passport').get('ldap').get, '')
# email = filters.pop(CONFIG.y('passport').get('ldap').get, '')
email = filters.pop('email')
user_dn = self.lookup(self.generate_filter(**{LOGIN_FIELD: email}))
if not user_dn:

86
passbook/lib/config.py
View File

@ -1,31 +1,35 @@
"""passbook lib config loader"""
"""passbook core config loader"""
import os
from collections.abc import Mapping
from collections import Mapping
from contextlib import contextmanager
from glob import glob
from logging import getLogger
from typing import Any
from urllib.parse import urlparse
import yaml
from django.conf import ImproperlyConfigured
from django.utils.autoreload import autoreload_started
from structlog import get_logger
SEARCH_PATHS = [
'passbook/lib/default.yml',
'/etc/passbook/config.yml',
'.',
'',
] + glob('/etc/passbook/config.d/*.yml', recursive=True)
LOGGER = getLogger(__name__)
ENVIRONMENT = os.getenv('PASSBOOK_ENV', 'local')
LOGGER = get_logger()
ENV_PREFIX = 'PASSBOOK'
ENVIRONMENT = os.getenv(f'{ENV_PREFIX}_ENV', 'local')
class ConfigLoader:
"""Search through SEARCH_PATHS and load configuration"""
"""Search through SEARCH_PATHS and load configuration. Environment variables starting with
`ENV_PREFIX` are also applied.
A variable like PASSBOOK_POSTGRESQL__HOST would translate to postgresql.host"""
loaded_file = []
__config = {}
__context_default = None
__sub_dicts = []
def __init__(self):
@ -47,15 +51,7 @@ class ConfigLoader:
if os.path.isfile(env_file) and os.path.exists(env_file):
# Update config with env file
self.update_from_file(env_file)
self.handle_secret_key()
def handle_secret_key(self):
"""Handle `secret_key_file`"""
if 'secret_key_file' in self.__config:
secret_key_file = self.__config.get('secret_key_file')
if os.path.isfile(secret_key_file) and os.path.exists(secret_key_file):
with open(secret_key_file) as file:
self.__config['secret_key'] = file.read().replace('\n', '')
self.update_from_env()
def update(self, root, updatee):
"""Recursively update dictionary"""
@ -63,16 +59,25 @@ class ConfigLoader:
if isinstance(value, Mapping):
root[key] = self.update(root.get(key, {}), value)
else:
if isinstance(value, str):
value = self.parse_uri(value)
root[key] = value
return root
def parse_uri(self, value):
"""Parse string values which start with a URI"""
url = urlparse(value)
if url.scheme == 'env':
value = os.getenv(url.netloc, url.query)
return value
def update_from_file(self, path: str):
"""Update config from file contents"""
try:
with open(path) as file:
try:
self.update(self.__config, yaml.safe_load(file))
LOGGER.debug("Loaded %s", path)
LOGGER.debug("Loaded config", file=path)
self.loaded_file.append(path)
except yaml.YAMLError as exc:
raise ImproperlyConfigured from exc
@ -83,12 +88,26 @@ class ConfigLoader:
"""Update config from dict"""
self.__config.update(update)
@contextmanager
def default(self, value: Any):
"""Contextmanage that sets default"""
self.__context_default = value
yield
self.__context_default = None
def update_from_env(self):
"""Check environment variables"""
outer = {}
idx = 0
for key, value in os.environ.items():
if not key.startswith(ENV_PREFIX):
continue
relative_key = key.replace(f"{ENV_PREFIX}_", '').replace('__', '.').lower()
# Recursively convert path from a.b.c into outer[a][b][c]
current_obj = outer
dot_parts = relative_key.split('.')
for dot_part in dot_parts[:-1]:
if dot_part not in current_obj:
current_obj[dot_part] = {}
current_obj = current_obj[dot_part]
current_obj[dot_parts[-1]] = value
idx += 1
if idx > 0:
LOGGER.debug("Loaded environment variables", count=idx)
self.update(self.__config, outer)
@contextmanager
# pylint: disable=invalid-name
@ -98,15 +117,6 @@ class ConfigLoader:
yield
self.__sub_dicts.pop()
def get(self, key: str, default=None) -> Any:
"""Get value from loaded config file"""
if default is None:
default = self.__context_default
config_copy = self.raw
for sub in self.__sub_dicts:
config_copy = config_copy.get(sub, None)
return config_copy.get(key, default)
@property
def raw(self) -> dict:
"""Get raw config dictionary"""
@ -115,8 +125,6 @@ class ConfigLoader:
# pylint: disable=invalid-name
def y(self, path: str, default=None, sep='.') -> Any:
"""Access attribute by using yaml path"""
if default is None:
default = self.__context_default
# Walk sub_dicts before parsing path
root = self.raw
for sub in self.__sub_dicts:
@ -129,11 +137,17 @@ class ConfigLoader:
return default
return root
def y_bool(self, path: str, default=False) -> bool:
"""Wrapper for y that converts value into boolean"""
return str(self.y(path, default)).lower() == 'true'
CONFIG = ConfigLoader()
# pylint: disable=unused-argument
def signal_handler(sender, **kwargs):
def signal_handler(sender, **_):
"""Add all loaded config files to autoreload watcher"""
for path in CONFIG.loaded_file:
sender.watch_file(path)

43
passbook/lib/default.yml
View File

@ -1,43 +1,20 @@
# This is the default configuration file
databases:
default:
engine: 'django.db.backends.postgresql'
postgresql:
host: localhost
name: passbook
user: passbook
password: 'EK-5jnKfjrGRm<77'
host: localhost
log:
level:
console: DEBUG
file: DEBUG
file: /dev/null
syslog:
host: 127.0.0.1
port: 514
email:
host: localhost
port: 25
user: ''
user: postgres
password: ''
use_tls: false
use_ssl: false
from: passbook <passbook@domain.tld>
web:
server.socket_host: 0.0.0.0
server.socket_port: 8000
server.thread_pool: 20
log.screen: false
log.access_file: ''
log.error_file: ''
redis:
host: localhost
password: ''
cache_db: 0
message_queue_db: 1
debug: false
secure_proxy_header:
HTTP_X_FORWARDED_PROTO: https
rabbitmq: guest:guest@localhost/passbook
redis: localhost/0
# Error reporting, sends stacktrace to sentry.services.beryju.org
error_report_enabled: true
secret_key: 9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s
domains:
- passbook.local

2
passbook/otp/views.py
View File

@ -75,7 +75,7 @@ class EnableView(LoginRequiredMixin, FormView):
# TODO: Check if OTP Factor exists and applies to user
def get_context_data(self, **kwargs):
kwargs['config'] = CONFIG.get('passbook')
kwargs['config'] = CONFIG.y('passbook')
kwargs['is_login'] = True
kwargs['title'] = _('Configue OTP')
kwargs['primary_action'] = _('Setup')

195
passbook/root/settings.py
View File

@ -15,19 +15,15 @@ import logging
import os
import sys
from celery.schedules import crontab
from django.contrib import messages
import structlog
from sentry_sdk import init as sentry_init
from sentry_sdk.integrations.celery import CeleryIntegration
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.logging import LoggingIntegration
from passbook import __version__
from passbook.lib.config import CONFIG
from passbook.lib.sentry import before_send
VERSION = __version__
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
STATIC_ROOT = BASE_DIR + '/static'
@ -36,12 +32,13 @@ STATIC_ROOT = BASE_DIR + '/static'
# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = CONFIG.get('secret_key')
SECRET_KEY = CONFIG.y('secret_key',
"9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s") # noqa Debug
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = CONFIG.get('debug')
DEBUG = CONFIG.y_bool('debug')
INTERNAL_IPS = ['127.0.0.1']
# ALLOWED_HOSTS = CONFIG.get('domains', []) + [CONFIG.get('primary_domain')]
# ALLOWED_HOSTS = CONFIG.y('domains', []) + [CONFIG.y('primary_domain')]
ALLOWED_HOSTS = ['*']
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
@ -53,7 +50,7 @@ AUTH_USER_MODEL = 'passbook_core.User'
CSRF_COOKIE_NAME = 'passbook_csrf'
SESSION_COOKIE_NAME = 'passbook_session'
SESSION_COOKIE_DOMAIN = CONFIG.get('primary_domain')
SESSION_COOKIE_DOMAIN = CONFIG.y('primary_domain')
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"
LANGUAGE_COOKIE_NAME = 'passbook_language'
@ -72,8 +69,8 @@ INSTALLED_APPS = [
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.postgres',
'rest_framework',
'drf_yasg',
# 'rest_framework',
# 'drf_yasg',
'passbook.core.apps.PassbookCoreConfig',
'passbook.admin.apps.PassbookAdminConfig',
'passbook.api.apps.PassbookAPIConfig',
@ -93,16 +90,6 @@ INSTALLED_APPS = [
'passbook.app_gw.apps.PassbookApplicationApplicationGatewayConfig',
]
# Message Tag fix for bootstrap CSS Classes
MESSAGE_TAGS = {
messages.DEBUG: 'primary',
messages.INFO: 'info',
messages.SUCCESS: 'success',
messages.WARNING: 'warning',
messages.ERROR: 'danger',
}
REST_FRAMEWORK = {
# Use Django's standard `django.contrib.auth` permissions,
# or allow read-only access for unauthenticated users.
@ -114,17 +101,20 @@ REST_FRAMEWORK = {
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://%s" % CONFIG.get('redis'),
"LOCATION": f"redis://{CONFIG.y('redis.host')}:6379/{CONFIG.y('redis.cache_db')}",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
}
}
}
DJANGO_REDIS_IGNORE_EXCEPTIONS = True
DJANGO_REDIS_LOG_IGNORED_EXCEPTIONS = True
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"
MIDDLEWARE = [
'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'passbook.app_gw.middleware.ApplicationGatewayMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
@ -150,21 +140,19 @@ TEMPLATES = [
},
]
WSGI_APPLICATION = 'passbook.core.wsgi.application'
WSGI_APPLICATION = 'passbook.root.wsgi.application'
# Database
# https://docs.djangoproject.com/en/2.1/ref/settings/#databases
DATABASES = {}
for db_alias, db_config in CONFIG.get('databases').items():
DATABASES[db_alias] = {
'ENGINE': db_config.get('engine'),
'HOST': db_config.get('host'),
'NAME': db_config.get('name'),
'USER': db_config.get('user'),
'PASSWORD': db_config.get('password'),
'OPTIONS': db_config.get('options', {}),
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'HOST': CONFIG.y('postgresql.host'),
'NAME': CONFIG.y('postgresql.name'),
'USER': CONFIG.y('postgresql.user'),
'PASSWORD': CONFIG.y('postgresql.password'),
}
}
# Password validation
@ -203,20 +191,13 @@ USE_TZ = True
# Celery settings
# Add a 10 minute timeout to all Celery tasks.
CELERY_TASK_SOFT_TIME_LIMIT = 600
CELERY_TIMEZONE = TIME_ZONE
CELERY_BEAT_SCHEDULE = {}
CELERY_CREATE_MISSING_QUEUES = True
CELERY_TASK_DEFAULT_QUEUE = 'passbook'
CELERY_BROKER_URL = 'amqp://%s' % CONFIG.get('rabbitmq')
CELERY_RESULT_BACKEND = 'rpc://'
CELERY_ACKS_LATE = True
CELERY_BROKER_HEARTBEAT = 0
CELERY_BEAT_SCHEDULE = {
'cleanup-expired-nonces': {
'task': 'passbook.core.tasks.clean_nonces',
'schedule': crontab(hour=1, minute=1)
}
}
CELERY_BROKER_URL = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}"
f":6379/{CONFIG.y('redis.message_queue_db')}")
CELERY_RESULT_BACKEND = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}"
f":6379/{CONFIG.y('redis.message_queue_db')}")
if not DEBUG:
@ -224,11 +205,7 @@ if not DEBUG:
dsn="https://33cdbcb23f8b436dbe0ee06847410b67@sentry.beryju.org/3",
integrations=[
DjangoIntegration(),
CeleryIntegration(),
LoggingIntegration(
level=logging.INFO,
event_level=logging.ERROR
)
CeleryIntegration()
],
send_default_pii=True,
before_send=before_send,
@ -240,95 +217,76 @@ if not DEBUG:
STATIC_URL = '/static/'
structlog.configure_once(
processors=[
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(),
structlog.processors.StackInfoRenderer(),
# structlog.processors.format_exc_info,
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
context_class=structlog.threadlocal.wrap_dict(dict),
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
LOG_PRE_CHAIN = [
# Add the log level and a timestamp to the event_dict if the log entry
# is not from structlog.
structlog.stdlib.add_log_level,
structlog.processors.TimeStamper(),
]
with CONFIG.cd('log'):
LOGGING_HANDLER_MAP = {
'passbook': 'DEBUG',
'django': 'WARNING',
'celery': 'WARNING',
'grpc': 'DEBUG',
'oauthlib': 'DEBUG',
'oauth2_provider': 'DEBUG',
'daphne': 'INFO',
}
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': ('%(asctime)s %(levelname)-8s %(name)-55s '
'%(funcName)-20s %(message)s'),
"plain": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": structlog.processors.JSONRenderer(),
"foreign_pre_chain": LOG_PRE_CHAIN,
},
'color': {
'()': 'colorlog.ColoredFormatter',
'format': ('%(log_color)s%(asctime)s %(levelname)-8s %(name)-55s '
'%(funcName)-20s %(message)s'),
'log_colors': {
'DEBUG': 'bold_black',
'INFO': 'white',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
'SUCCESS': 'green',
"colored": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": structlog.dev.ConsoleRenderer(colors=DEBUG),
"foreign_pre_chain": LOG_PRE_CHAIN,
},
}
},
'handlers': {
'console': {
'level': CONFIG.get('level').get('console'),
'level': DEBUG,
'class': 'logging.StreamHandler',
'formatter': 'color',
},
'syslog': {
'level': CONFIG.get('level').get('file'),
'class': 'logging.handlers.SysLogHandler',
'formatter': 'verbose',
'address': (CONFIG.get('syslog').get('host'),
CONFIG.get('syslog').get('port'))
},
'file': {
'level': CONFIG.get('level').get('file'),
'class': 'logging.FileHandler',
'formatter': 'verbose',
'filename': CONFIG.get('file'),
'formatter': "colored" if DEBUG else "plain",
},
'queue': {
'level': CONFIG.get('level').get('console'),
'level': DEBUG,
'class': 'passbook.lib.log.QueueListenerHandler',
'handlers': [
'cfg://handlers.console',
# 'cfg://handlers.syslog',
'cfg://handlers.file',
],
}
},
'loggers': {
'passbook': {
'handlers': ['queue'],
'level': 'DEBUG',
'propagate': True,
},
'django': {
'handlers': ['queue'],
'level': 'INFO',
'propagate': True,
},
'tasks': {
'handlers': ['queue'],
'level': 'DEBUG',
'propagate': True,
},
'cherrypy': {
'handlers': ['queue'],
'level': 'DEBUG',
'propagate': True,
},
'oauthlib': {
'handlers': ['queue'],
'level': 'DEBUG',
'propagate': True,
},
'oauth2_provider': {
'handlers': ['queue'],
'level': 'DEBUG',
'propagate': True,
},
'daphne': {
'handlers': ['queue'],
'level': 'INFO',
'propagate': True,
}
}
for handler_name, level in LOGGING_HANDLER_MAP.items():
LOGGING['loggers'][handler_name] = {
'handlers': ['console'],
'level': level,
'propagate': True,
}
TEST = False
@ -342,6 +300,7 @@ if any('test' in arg for arg in sys.argv):
TEST = True
CELERY_TASK_ALWAYS_EAGER = True
_DISALLOWED_ITEMS = ['INSTALLED_APPS', 'MIDDLEWARE', 'AUTHENTICATION_BACKENDS']
# Load subapps's INSTALLED_APPS
for _app in INSTALLED_APPS: