From 0c2e9234bf0d845ef0e56d98c43d88f63d305243 Mon Sep 17 00:00:00 2001 From: Jens L Date: Mon, 4 Apr 2022 14:02:10 +0200 Subject: [PATCH] ci: use native caching (#2665) * ci: use native caching * migrate to actions * cleanup * migrate more --- .github/actions/docker-setup/action.yml | 49 ++++++++ .github/actions/setup/action.yml | 45 +++++++ .../actions/setup/docker-compose.yml | 0 .github/workflows/ci-main.yml | 119 ++++-------------- .github/workflows/ci-outpost.yml | 3 +- .github/workflows/translation-compile.yml | 15 +-- scripts/ci_prepare.sh | 12 -- scripts/generate_ci_config.py | 13 -- scripts/gh_env.py | 22 ---- 9 files changed, 119 insertions(+), 159 deletions(-) create mode 100644 .github/actions/docker-setup/action.yml create mode 100644 .github/actions/setup/action.yml rename scripts/ci.docker-compose.yml => .github/actions/setup/docker-compose.yml (100%) delete mode 100755 scripts/ci_prepare.sh delete mode 100644 scripts/generate_ci_config.py delete mode 100644 scripts/gh_env.py diff --git a/.github/actions/docker-setup/action.yml b/.github/actions/docker-setup/action.yml new file mode 100644 index 000000000..bba66edab --- /dev/null +++ b/.github/actions/docker-setup/action.yml @@ -0,0 +1,49 @@ +name: 'Prepare docker environment variables' +description: 'Prepare docker environment variables' + +outputs: + shouldBuild: + description: "Whether to build image or not" + value: ${{ steps.ev.outputs.shouldBuild }} + branchName: + description: "Branch name" + value: ${{ steps.ev.outputs.branchName }} + branchNameContainer: + description: "Branch name (for containers)" + value: ${{ steps.ev.outputs.branchNameContainer }} + timestamp: + description: "Timestamp" + value: ${{ steps.ev.outputs.timestamp }} + sha: + description: "sha" + value: ${{ steps.ev.outputs.sha }} + +runs: + using: "composite" + steps: + - name: Generate config + id: ev + shell: python + run: | + """Helper script to get the actual branch name, docker safe""" + import os + from time import time + + env_pr_branch = "GITHUB_HEAD_REF" + default_branch = "GITHUB_REF" + sha = "GITHUB_SHA" + + branch_name = os.environ[default_branch] + if os.environ.get(env_pr_branch, "") != "": + branch_name = os.environ[env_pr_branch] + + should_build = str(os.environ.get("DOCKER_USERNAME", "") != "").lower() + + print("##[set-output name=branchName]%s" % branch_name) + print( + "##[set-output name=branchNameContainer]%s" + % branch_name.replace("refs/heads/", "").replace("/", "-") + ) + print("##[set-output name=timestamp]%s" % int(time())) + print("##[set-output name=sha]%s" % os.environ[sha]) + print("##[set-output name=shouldBuild]%s" % should_build) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 000000000..428ff9d7a --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,45 @@ +name: 'Setup authentik testing environemnt' +description: 'Setup authentik testing environemnt' + +runs: + using: "composite" + steps: + - name: Install poetry + shell: bash + run: | + pipx install poetry || true + sudo apt update + sudo apt install -y libxmlsec1-dev pkg-config gettext + - name: Setup python and restore poetry + uses: actions/setup-python@v3 + with: + python-version: '3.10' + cache: 'poetry' + - name: Setup node + uses: actions/setup-node@v3.1.0 + with: + node-version: '16' + cache: 'npm' + cache-dependency-path: web/package-lock.json + - name: Setup dependencies + shell: bash + run: | + docker-compose -f .github/actions/setup/docker-compose.yml up -d + poetry env use python3.10 + poetry install + npm install -g pyright@1.1.136 + - name: Generate config + shell: poetry run python {0} + run: | + from authentik.lib.generators import generate_id + from yaml import safe_dump + + with open("local.env.yml", "w") as _config: + safe_dump( + { + "log_level": "debug", + "secret_key": generate_id(), + }, + _config, + default_flow_style=False, + ) diff --git a/scripts/ci.docker-compose.yml b/.github/actions/setup/docker-compose.yml similarity index 100% rename from scripts/ci.docker-compose.yml rename to .github/actions/setup/docker-compose.yml diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index b6de65252..9b4a83d2f 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -32,35 +32,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: actions/setup-node@v3.1.0 - with: - node-version: '16' - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} - run: scripts/ci_prepare.sh + - name: Setup authentik env + uses: ./.github/actions/setup - name: run job run: poetry run make ci-${{ matrix.job }} test-migrations: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} - run: scripts/ci_prepare.sh + - name: Setup authentik env + uses: ./.github/actions/setup - name: run migrations run: poetry run python -m lifecycle.migrate test-migrations-from-stable: @@ -69,17 +50,8 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-python@v3 - - name: prepare variables - id: ev - run: | - python ./scripts/gh_env.py - sudo pip install -U pipenv - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} + - name: Setup authentik env + uses: ./.github/actions/setup - name: checkout stable run: | # Copy current, latest config to local @@ -89,13 +61,8 @@ jobs: git checkout $(git describe --abbrev=0 --match 'version/*') rm -rf .github/ scripts/ mv ../.github ../scripts . - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} - run: | - scripts/ci_prepare.sh - # install anyways since stable will have different dependencies - poetry install + - name: Setup authentik env (ensure stable deps are installed) + uses: ./.github/actions/setup - name: run migrations to stable run: poetry run python -m lifecycle.migrate - name: checkout current code @@ -103,28 +70,19 @@ jobs: set -x git fetch git reset --hard HEAD + git clean -d -fx . git checkout $GITHUB_SHA poetry install - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} - run: scripts/ci_prepare.sh + - name: Setup authentik env (ensure latest deps are installed) + uses: ./.github/actions/setup - name: migrate to latest run: poetry run python -m lifecycle.migrate test-unittest: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} - run: scripts/ci_prepare.sh + - name: Setup authentik env + uses: ./.github/actions/setup - uses: testspace-com/setup-testspace@v1 with: domain: ${{github.repository_owner}} @@ -142,16 +100,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} - run: scripts/ci_prepare.sh + - name: Setup authentik env + uses: ./.github/actions/setup - uses: testspace-com/setup-testspace@v1 with: domain: ${{github.repository_owner}} @@ -171,25 +121,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: actions/setup-node@v3.1.0 - with: - node-version: '16' - cache: 'npm' - cache-dependency-path: web/package-lock.json + - name: Setup authentik env + uses: ./.github/actions/setup - uses: testspace-com/setup-testspace@v1 with: domain: ${{github.repository_owner}} - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} + - name: Setup authentik env run: | - scripts/ci_prepare.sh docker-compose -f tests/e2e/docker-compose.yml up -d - id: cache-web uses: actions/cache@v3 @@ -216,25 +154,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: actions/setup-node@v3.1.0 - with: - node-version: '16' - cache: 'npm' - cache-dependency-path: web/package-lock.json + - name: Setup authentik env + uses: ./.github/actions/setup - uses: testspace-com/setup-testspace@v1 with: domain: ${{github.repository_owner}} - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} + - name: Setup authentik env run: | - scripts/ci_prepare.sh docker-compose -f tests/e2e/docker-compose.yml up -d - id: cache-web uses: actions/cache@v3 @@ -288,8 +214,7 @@ jobs: id: ev env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - run: | - python ./scripts/gh_env.py + uses: ./.github/actions/docker-setup - name: Login to Container Registry uses: docker/login-action@v1 if: ${{ steps.ev.outputs.shouldBuild == 'true' }} diff --git a/.github/workflows/ci-outpost.yml b/.github/workflows/ci-outpost.yml index 1377e957e..024bc45df 100644 --- a/.github/workflows/ci-outpost.yml +++ b/.github/workflows/ci-outpost.yml @@ -68,10 +68,9 @@ jobs: uses: docker/setup-buildx-action@v1 - name: prepare variables id: ev + uses: ./.github/actions/docker-setup env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - run: | - python ./scripts/gh_env.py - name: Login to Container Registry uses: docker/login-action@v1 if: ${{ steps.ev.outputs.shouldBuild == 'true' }} diff --git a/.github/workflows/translation-compile.yml b/.github/workflows/translation-compile.yml index 64b1982c1..68483dd6e 100644 --- a/.github/workflows/translation-compile.yml +++ b/.github/workflows/translation-compile.yml @@ -21,19 +21,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - id: cache-poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-cache-v2-${{ hashFiles('**/poetry.lock') }} - - name: prepare - env: - INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }} - run: | - sudo apt-get update - sudo apt-get install -y gettext - scripts/ci_prepare.sh + - name: Setup authentik env + uses: ./.github/actions/setup - name: run compile run: poetry run ./manage.py compilemessages - name: Create Pull Request diff --git a/scripts/ci_prepare.sh b/scripts/ci_prepare.sh deleted file mode 100755 index a523dd431..000000000 --- a/scripts/ci_prepare.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -xe -docker-compose -f scripts/ci.docker-compose.yml up -d - -sudo apt update -sudo apt install -y libxmlsec1-dev pkg-config -python3 -m pip install -U wheel poetry -poetry env use python3.10 -if [[ "$INSTALL" != "true" ]]; then - poetry install -fi -poetry run python -m scripts.generate_ci_config -npm install -g pyright@1.1.136 diff --git a/scripts/generate_ci_config.py b/scripts/generate_ci_config.py deleted file mode 100644 index 5a0c89ee7..000000000 --- a/scripts/generate_ci_config.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Utility script to generate a config for CI runs""" -from authentik.lib.generators import generate_id -from yaml import safe_dump - -with open("local.env.yml", "w") as _config: - safe_dump( - { - "log_level": "debug", - "secret_key": generate_id(), - }, - _config, - default_flow_style=False, - ) diff --git a/scripts/gh_env.py b/scripts/gh_env.py deleted file mode 100644 index 7917aabe9..000000000 --- a/scripts/gh_env.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Helper script to get the actual branch name, docker safe""" -import os -from time import time - -env_pr_branch = "GITHUB_HEAD_REF" -default_branch = "GITHUB_REF" -sha = "GITHUB_SHA" - -branch_name = os.environ[default_branch] -if os.environ.get(env_pr_branch, "") != "": - branch_name = os.environ[env_pr_branch] - -should_build = str(os.environ.get("DOCKER_USERNAME", "") != "").lower() - -print("##[set-output name=branchName]%s" % branch_name) -print( - "##[set-output name=branchNameContainer]%s" - % branch_name.replace("refs/heads/", "").replace("/", "-") -) -print("##[set-output name=timestamp]%s" % int(time())) -print("##[set-output name=sha]%s" % os.environ[sha]) -print("##[set-output name=shouldBuild]%s" % should_build)