diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..f944db09f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,27 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: PDB attach Server", + "type": "python", + "request": "attach", + "connect": { + "host": "localhost", + "port": 6800 + }, + "justMyCode": true, + "django": true + }, + { + "name": "Python: PDB attach Worker", + "type": "python", + "request": "attach", + "connect": { + "host": "localhost", + "port": 6900 + }, + "justMyCode": true, + "django": true + }, + ] +} diff --git a/authentik/admin/api/workers.py b/authentik/admin/api/workers.py index cfb23ea31..ab6d03873 100644 --- a/authentik/admin/api/workers.py +++ b/authentik/admin/api/workers.py @@ -19,7 +19,7 @@ class WorkerView(APIView): def get(self, request: Request) -> Response: """Get currently connected worker count.""" count = len(CELERY_APP.control.ping(timeout=0.5)) - # In debug we run with `CELERY_TASK_ALWAYS_EAGER`, so tasks are ran on the main process + # In debug we run with `task_always_eager`, so tasks are ran on the main process if settings.DEBUG: # pragma: no cover count += 1 return Response({"count": count}) diff --git a/authentik/core/management/commands/worker.py b/authentik/core/management/commands/worker.py new file mode 100644 index 000000000..ad9fbe5c9 --- /dev/null +++ b/authentik/core/management/commands/worker.py @@ -0,0 +1,40 @@ +"""Run worker""" +from sys import exit as sysexit +from tempfile import tempdir + +from celery.apps.worker import Worker +from django.core.management.base import BaseCommand +from django.db import close_old_connections +from structlog.stdlib import get_logger + +from authentik.lib.config import CONFIG +from authentik.root.celery import CELERY_APP + +LOGGER = get_logger() + + +class Command(BaseCommand): + """Run worker""" + + def handle(self, **options): + close_old_connections() + if CONFIG.y_bool("remote_debug"): + import debugpy + + debugpy.listen(("0.0.0.0", 6900)) # nosec + worker: Worker = CELERY_APP.Worker( + no_color=False, + quiet=True, + optimization="fair", + max_tasks_per_child=1, + autoscale=(3, 1), + task_events=True, + beat=True, + schedule_filename=f"{tempdir}/celerybeat-schedule", + queues=["authentik", "authentik_scheduled", "authentik_events"], + ) + for task in CELERY_APP.tasks: + LOGGER.debug("Registered task", task=task) + + worker.start() + sysexit(worker.exitcode) diff --git a/authentik/events/monitored_tasks.py b/authentik/events/monitored_tasks.py index 36641506b..5db4febef 100644 --- a/authentik/events/monitored_tasks.py +++ b/authentik/events/monitored_tasks.py @@ -41,6 +41,7 @@ class TaskResult: def with_error(self, exc: Exception) -> "TaskResult": """Since errors might not always be pickle-able, set the traceback""" + # TODO: Mark exception somehow so that is rendered as
in frontend self.messages.append(exception_to_string(exc)) return self diff --git a/authentik/lib/default.yml b/authentik/lib/default.yml index c6d620315..39e55c258 100644 --- a/authentik/lib/default.yml +++ b/authentik/lib/default.yml @@ -26,6 +26,7 @@ redis: cache_timeout_reputation: 300 debug: false +remote_debug: false log_level: info diff --git a/authentik/root/celery.py b/authentik/root/celery.py index 136e004fa..e9293b58d 100644 --- a/authentik/root/celery.py +++ b/authentik/root/celery.py @@ -130,11 +130,7 @@ class LivenessProbe(bootsteps.StartStopStep): HEARTBEAT_FILE.touch() -# Using a string here means the worker doesn't have to serialize -# the configuration object to child processes. -# - namespace='CELERY' means all celery-related configuration keys -# should have a `CELERY_` prefix. -CELERY_APP.config_from_object(settings, namespace="CELERY") +CELERY_APP.config_from_object(settings.CELERY) # Load task modules from all registered Django app configs. CELERY_APP.autodiscover_tasks() diff --git a/authentik/root/settings.py b/authentik/root/settings.py index 5abb8d18f..99918512a 100644 --- a/authentik/root/settings.py +++ b/authentik/root/settings.py @@ -182,13 +182,13 @@ REST_FRAMEWORK = { }, } -REDIS_PROTOCOL_PREFIX = "redis://" -REDIS_CELERY_TLS_REQUIREMENTS = "" +_redis_protocol_prefix = "redis://" +_redis_celery_tls_requirements = "" if CONFIG.y_bool("redis.tls", False): - REDIS_PROTOCOL_PREFIX = "rediss://" - REDIS_CELERY_TLS_REQUIREMENTS = f"?ssl_cert_reqs={CONFIG.y('redis.tls_reqs')}" + _redis_protocol_prefix = "rediss://" + _redis_celery_tls_requirements = f"?ssl_cert_reqs={CONFIG.y('redis.tls_reqs')}" _redis_url = ( - f"{REDIS_PROTOCOL_PREFIX}:" + f"{_redis_protocol_prefix}:" f"{quote_plus(CONFIG.y('redis.password'))}@{quote_plus(CONFIG.y('redis.host'))}:" f"{int(CONFIG.y('redis.port'))}" ) @@ -326,27 +326,27 @@ USE_TZ = True LOCALE_PATHS = ["./locale"] -# Celery settings -# Add a 10 minute timeout to all Celery tasks. -CELERY_TASK_SOFT_TIME_LIMIT = 600 -CELERY_WORKER_MAX_TASKS_PER_CHILD = 50 -CELERY_WORKER_CONCURRENCY = 2 -CELERY_BEAT_SCHEDULE = { - "clean_expired_models": { - "task": "authentik.core.tasks.clean_expired_models", - "schedule": crontab(minute="2-59/5"), - "options": {"queue": "authentik_scheduled"}, - }, - "user_cleanup": { - "task": "authentik.core.tasks.clean_temporary_users", - "schedule": crontab(minute="9-59/5"), - "options": {"queue": "authentik_scheduled"}, +CELERY = { + "task_soft_time_limit": 600, + "worker_max_tasks_per_child": 50, + "worker_concurrency": 2, + "beat_schedule": { + "clean_expired_models": { + "task": "authentik.core.tasks.clean_expired_models", + "schedule": crontab(minute="2-59/5"), + "options": {"queue": "authentik_scheduled"}, + }, + "user_cleanup": { + "task": "authentik.core.tasks.clean_temporary_users", + "schedule": crontab(minute="9-59/5"), + "options": {"queue": "authentik_scheduled"}, + }, }, + "task_create_missing_queues": True, + "task_default_queue": "authentik", + "broker_url": f"{_redis_url}/{CONFIG.y('redis.db')}{_redis_celery_tls_requirements}", + "result_backend": f"{_redis_url}/{CONFIG.y('redis.db')}{_redis_celery_tls_requirements}", } -CELERY_TASK_CREATE_MISSING_QUEUES = True -CELERY_TASK_DEFAULT_QUEUE = "authentik" -CELERY_BROKER_URL = f"{_redis_url}/{CONFIG.y('redis.db')}{REDIS_CELERY_TLS_REQUIREMENTS}" -CELERY_RESULT_BACKEND = f"{_redis_url}/{CONFIG.y('redis.db')}{REDIS_CELERY_TLS_REQUIREMENTS}" # Sentry integration env = get_env() @@ -455,7 +455,7 @@ _DISALLOWED_ITEMS = [ "INSTALLED_APPS", "MIDDLEWARE", "AUTHENTICATION_BACKENDS", - "CELERY_BEAT_SCHEDULE", + "CELERY", ] @@ -466,7 +466,7 @@ def _update_settings(app_path: str): INSTALLED_APPS.extend(getattr(settings_module, "INSTALLED_APPS", [])) MIDDLEWARE.extend(getattr(settings_module, "MIDDLEWARE", [])) AUTHENTICATION_BACKENDS.extend(getattr(settings_module, "AUTHENTICATION_BACKENDS", [])) - CELERY_BEAT_SCHEDULE.update(getattr(settings_module, "CELERY_BEAT_SCHEDULE", {})) + CELERY["beat_schedule"].update(getattr(settings_module, "CELERY_BEAT_SCHEDULE", {})) for _attr in dir(settings_module): if not _attr.startswith("__") and _attr not in _DISALLOWED_ITEMS: globals()[_attr] = getattr(settings_module, _attr) @@ -482,7 +482,7 @@ for _app in INSTALLED_APPS: _update_settings("data.user_settings") if DEBUG: - CELERY_TASK_ALWAYS_EAGER = True + CELERY["task_always_eager"] = True os.environ[ENV_GIT_HASH_KEY] = "dev" INSTALLED_APPS.append("silk") SILKY_PYTHON_PROFILER = True diff --git a/authentik/root/test_runner.py b/authentik/root/test_runner.py index 8f00ff85e..7a420c157 100644 --- a/authentik/root/test_runner.py +++ b/authentik/root/test_runner.py @@ -30,7 +30,7 @@ class PytestTestRunner: # pragma: no cover self.args.append(f"--randomly-seed={kwargs['randomly_seed']}") settings.TEST = True - settings.CELERY_TASK_ALWAYS_EAGER = True + settings.CELERY["task_always_eager"] = True CONFIG.y_set("avatars", "none") CONFIG.y_set("geoip", "tests/GeoLite2-City-Test.mmdb") CONFIG.y_set("blueprints_dir", "./blueprints") diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 000000000..973659a09 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,36 @@ +# This file is used for development and debugging, and should not be used for production instances + +version: '3.5' + +services: + flower: + image: ${AUTHENTIK_IMAGE:-ghcr.io/goauthentik/server}:${AUTHENTIK_TAG:-2023.5.4} + restart: unless-stopped + command: worker-status + environment: + AUTHENTIK_REDIS__HOST: redis + AUTHENTIK_POSTGRESQL__HOST: postgresql + AUTHENTIK_POSTGRESQL__USER: ${PG_USER:-authentik} + AUTHENTIK_POSTGRESQL__NAME: ${PG_DB:-authentik} + AUTHENTIK_POSTGRESQL__PASSWORD: ${PG_PASS} + env_file: + - .env + ports: + - "9001:9000" + depends_on: + - postgresql + - redis + server: + environment: + AUTHENTIK_REMOTE_DEBUG: "true" + PYDEVD_THREAD_DUMP_ON_WARN_EVALUATION_TIMEOUT: "true" + ports: + - 6800:6800 + worker: + environment: + CELERY_RDB_HOST: "0.0.0.0" + CELERY_RDBSIG: "1" + AUTHENTIK_REMOTE_DEBUG: "true" + PYDEVD_THREAD_DUMP_ON_WARN_EVALUATION_TIMEOUT: "true" + ports: + - 6900:6900 diff --git a/lifecycle/ak b/lifecycle/ak index 97b1f523c..3b061cd52 100755 --- a/lifecycle/ak +++ b/lifecycle/ak @@ -54,6 +54,16 @@ function cleanup { rm -f ${MODE_FILE} } +function prepare_debug { + pip install --no-cache-dir -r /requirements-dev.txt + touch /unittest.xml + chown authentik:authentik /unittest.xml +} + +if [[ "${AUTHENTIK_REMOTE_DEBUG}" == "true" ]]; then + prepare_debug +fi + if [[ "$1" == "server" ]]; then wait_for_db set_mode "server" @@ -67,7 +77,7 @@ if [[ "$1" == "server" ]]; then elif [[ "$1" == "worker" ]]; then wait_for_db set_mode "worker" - check_if_root "celery -A authentik.root.celery worker -Ofair --max-tasks-per-child=1 --autoscale 3,1 -E -B -s /tmp/celerybeat-schedule -Q authentik,authentik_scheduled,authentik_events" + check_if_root "python -m manage worker" elif [[ "$1" == "worker-status" ]]; then wait_for_db celery -A authentik.root.celery flower \ @@ -75,9 +85,7 @@ elif [[ "$1" == "worker-status" ]]; then elif [[ "$1" == "bash" ]]; then /bin/bash elif [[ "$1" == "test-all" ]]; then - pip install --no-cache-dir -r /requirements-dev.txt - touch /unittest.xml - chown authentik:authentik /unittest.xml + prepare_debug check_if_root "python -m manage test authentik" elif [[ "$1" == "healthcheck" ]]; then run_authentik healthcheck $(cat $MODE_FILE) diff --git a/lifecycle/gunicorn.conf.py b/lifecycle/gunicorn.conf.py index 641e23528..f9c21ebdd 100644 --- a/lifecycle/gunicorn.conf.py +++ b/lifecycle/gunicorn.conf.py @@ -157,3 +157,8 @@ if not CONFIG.y_bool("disable_startup_analytics", False): # pylint: disable=broad-exception-caught except Exception: # nosec pass + +if CONFIG.y_bool("remote_debug"): + import debugpy + + debugpy.listen(("0.0.0.0", 6800)) # nosec diff --git a/poetry.lock b/poetry.lock index b92cb58f8..a60a54b77 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1066,6 +1066,33 @@ twisted = {version = ">=22.4", extras = ["tls"]} [package.extras] tests = ["django", "hypothesis", "pytest", "pytest-asyncio"] +[[package]] +name = "debugpy" +version = "1.6.7" +description = "An implementation of the Debug Adapter Protocol for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "debugpy-1.6.7-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b3e7ac809b991006ad7f857f016fa92014445085711ef111fdc3f74f66144096"}, + {file = "debugpy-1.6.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3876611d114a18aafef6383695dfc3f1217c98a9168c1aaf1a02b01ec7d8d1e"}, + {file = "debugpy-1.6.7-cp310-cp310-win32.whl", hash = "sha256:33edb4afa85c098c24cc361d72ba7c21bb92f501104514d4ffec1fb36e09c01a"}, + {file = "debugpy-1.6.7-cp310-cp310-win_amd64.whl", hash = "sha256:ed6d5413474e209ba50b1a75b2d9eecf64d41e6e4501977991cdc755dc83ab0f"}, + {file = "debugpy-1.6.7-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:38ed626353e7c63f4b11efad659be04c23de2b0d15efff77b60e4740ea685d07"}, + {file = "debugpy-1.6.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:279d64c408c60431c8ee832dfd9ace7c396984fd7341fa3116aee414e7dcd88d"}, + {file = "debugpy-1.6.7-cp37-cp37m-win32.whl", hash = "sha256:dbe04e7568aa69361a5b4c47b4493d5680bfa3a911d1e105fbea1b1f23f3eb45"}, + {file = "debugpy-1.6.7-cp37-cp37m-win_amd64.whl", hash = "sha256:f90a2d4ad9a035cee7331c06a4cf2245e38bd7c89554fe3b616d90ab8aab89cc"}, + {file = "debugpy-1.6.7-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:5224eabbbeddcf1943d4e2821876f3e5d7d383f27390b82da5d9558fd4eb30a9"}, + {file = "debugpy-1.6.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae1123dff5bfe548ba1683eb972329ba6d646c3a80e6b4c06cd1b1dd0205e9b"}, + {file = "debugpy-1.6.7-cp38-cp38-win32.whl", hash = "sha256:9cd10cf338e0907fdcf9eac9087faa30f150ef5445af5a545d307055141dd7a4"}, + {file = "debugpy-1.6.7-cp38-cp38-win_amd64.whl", hash = "sha256:aaf6da50377ff4056c8ed470da24632b42e4087bc826845daad7af211e00faad"}, + {file = "debugpy-1.6.7-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:0679b7e1e3523bd7d7869447ec67b59728675aadfc038550a63a362b63029d2c"}, + {file = "debugpy-1.6.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de86029696e1b3b4d0d49076b9eba606c226e33ae312a57a46dca14ff370894d"}, + {file = "debugpy-1.6.7-cp39-cp39-win32.whl", hash = "sha256:d71b31117779d9a90b745720c0eab54ae1da76d5b38c8026c654f4a066b0130a"}, + {file = "debugpy-1.6.7-cp39-cp39-win_amd64.whl", hash = "sha256:c0ff93ae90a03b06d85b2c529eca51ab15457868a377c4cc40a23ab0e4e552a3"}, + {file = "debugpy-1.6.7-py2.py3-none-any.whl", hash = "sha256:53f7a456bc50706a0eaabecf2d3ce44c4d5010e46dfc65b6b81a518b42866267"}, + {file = "debugpy-1.6.7.zip", hash = "sha256:c4c2f0810fa25323abfdfa36cbbbb24e5c3b1a42cb762782de64439c575d67f2"}, +] + [[package]] name = "deepmerge" version = "1.1.0" @@ -1686,13 +1713,13 @@ test = ["Cython (>=0.29.24,<0.30.0)"] [[package]] name = "humanize" -version = "4.6.0" +version = "4.7.0" description = "Python humanize utilities" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "humanize-4.6.0-py3-none-any.whl", hash = "sha256:401201aca462749773f02920139f302450cb548b70489b9b4b92be39fe3c3c50"}, - {file = "humanize-4.6.0.tar.gz", hash = "sha256:5f1f22bc65911eb1a6ffe7659bd6598e33dcfeeb904eb16ee1e705a09bf75916"}, + {file = "humanize-4.7.0-py3-none-any.whl", hash = "sha256:df7c429c2d27372b249d3f26eb53b07b166b661326e0325793e0a988082e3889"}, + {file = "humanize-4.7.0.tar.gz", hash = "sha256:7ca0e43e870981fa684acb5b062deb307218193bca1a01f2b2676479df849b3a"}, ] [package.extras] @@ -3419,13 +3446,13 @@ wsproto = ">=0.14" [[package]] name = "twilio" -version = "8.3.0" +version = "8.4.0" description = "Twilio API client and TwiML generator" optional = false python-versions = ">=3.7.0" files = [ - {file = "twilio-8.3.0-py2.py3-none-any.whl", hash = "sha256:f8f4a26e7491e015777c2c12abcc068321f12302d081fc355df486601434c311"}, - {file = "twilio-8.3.0.tar.gz", hash = "sha256:e76543b054f09304557d9bd0f9e3c21d09ca935d88f833788d43cab1f1fb67d1"}, + {file = "twilio-8.4.0-py2.py3-none-any.whl", hash = "sha256:56b812b4d77dabcfdf7aa02aac966065e064beabd083621940856a6ee0d060ee"}, + {file = "twilio-8.4.0.tar.gz", hash = "sha256:23fa599223d336a19d674394535d42bd1e260f7ca350a51d02b9d902370d76ef"}, ] [package.dependencies] @@ -4159,4 +4186,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "7c78d6909ba8cc5b8fb41233e2506f0b919b71e263213068e479af706a9670ce" +content-hash = "60a0e729895ebd44235e88e0414cc64e50c41736903ea61e6fb94a542dd2bb3c" diff --git a/pyproject.toml b/pyproject.toml index b60bbf71d..1b32f54cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,6 +178,7 @@ black = "*" bump2version = "*" colorama = "*" coverage = { extras = ["toml"], version = "*" } +debugpy = "*" django-silk = "*" drf-jsonschema-serializer = "*" importlib-metadata = "*"