Compare commits

...
This repository has been archived on 2024-05-31. You can view files and clone it, but cannot push or open issues or pull requests.

7 Commits

Author SHA1 Message Date
Jens Langhammer 241059f56b
replace ioutil
Signed-off-by: Jens Langhammer <jens@goauthentik.io>
2023-06-23 18:45:31 +02:00
Jens Langhammer 18db59ad2d
fix error comparison
Signed-off-by: Jens Langhammer <jens@goauthentik.io>
2023-06-23 18:44:25 +02:00
Marc 'risson' Schmitt d3ef158360
root: restart gunicorn on configuration changes
Signed-off-by: Marc 'risson' Schmitt <marc.schmitt@risson.space>
2023-04-29 01:32:14 +02:00
Marc 'risson' Schmitt 969ec188a7
root: config: implement config watching
Signed-off-by: Marc 'risson' Schmitt <marc.schmitt@risson.space>
2023-04-29 01:32:14 +02:00
Marc 'risson' Schmitt 2a15004232
root: config: remove redundant default configs
Signed-off-by: Marc 'risson' Schmitt <marc.schmitt@risson.space>
2023-04-29 01:32:14 +02:00
Marc 'risson' Schmitt 4e1d8543e3
root: config: config discovery parity between go and python
Signed-off-by: Marc 'risson' Schmitt <marc.schmitt@risson.space>
2023-04-29 00:39:34 +02:00
Marc 'risson' Schmitt fc5f6d6677
root: handle SIGHUP and SIGUSR2
This is the first step to handle configuration reloading. With those
changes, it is already possible to do so, by sending a SIGUSR2 signal to
the Go server process. The next step would be to watch for changes to
configuration files and call the Restart function of the GoUnicorn
instance.

SIGHUP is catched by the go server and forwarded as-is to gunicorn,
which causes it to restart its workers. However, that does not trigger
a reload of the Django settings, probably because they are already
loaded in the master, before creating any of the worker instances.

SIGUSR2, however, can be used to spawn a new gunicorn master process,
but handling it is a bit trickier. Please refer to Gunicorn's
documentation[0] for details, especially the "Upgrading to a new binary
on the fly" section.

As we are now effectively killing the gunicorn processed launched by the
server, we need to handle some sort of check to make sure it is still
running. That's done by using the already existing healthchecks, making
them useful not only for the application start, but also for its
lifetime. If a check is failed too many times in a given time period,
the gunicorn processed is killed (if necessary) and then restarted.

[0] https://docs.gunicorn.org/en/20.1.0/signals.html

Other relevant links and documentation:

Python library handling the processing swaping upon a SIGUSR2:
https://github.com/flupke/rainbow-saddle/

Golang cannot easily check if a process exists on Unix systems:
https://github.com/golang/go/issues/34396

Signed-off-by: Marc 'risson' Schmitt <marc.schmitt@risson.space>
2023-04-29 00:00:45 +02:00
7 changed files with 296 additions and 30 deletions

View File

@ -11,7 +11,11 @@ postgresql:
listen:
listen_http: 0.0.0.0:9000
listen_https: 0.0.0.0:9443
listen_ldap: 0.0.0.0:3389
listen_ldaps: 0.0.0.0:6636
listen_radius: 0.0.0.0:1812
listen_metrics: 0.0.0.0:9300
listen_debug: 0.0.0.0:9900
redis:
host: localhost
@ -25,6 +29,9 @@ redis:
cache_timeout_policies: 300
cache_timeout_reputation: 300
paths:
media: ./media
debug: false
log_level: info

View File

@ -4,11 +4,15 @@ import (
"fmt"
"net/http"
"net/url"
"os"
"os/signal"
"syscall"
"time"
"github.com/getsentry/sentry-go"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"goauthentik.io/internal/common"
"goauthentik.io/internal/config"
"goauthentik.io/internal/constants"
@ -70,6 +74,21 @@ var rootCmd = &cobra.Command{
l.Info("shutting down gunicorn")
g.Kill()
}()
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGUSR2)
go func() {
sig := <-c
if sig == syscall.SIGHUP {
log.Info("SIGHUP received, forwarding to gunicorn")
g.Reload()
}
if sig == syscall.SIGUSR2 {
log.Info("SIGUSR2 received, restarting gunicorn")
g.Restart()
}
}()
ws := web.NewWebServer(g)
g.HealthyCallback = func() {
if !config.Get().Outposts.DisableEmbeddedOutpost {
@ -78,6 +97,17 @@ var rootCmd = &cobra.Command{
}
go web.RunMetricsServer()
go attemptStartBackend(g)
w, err := config.WatchChanges(func() {
g.Restart()
})
if err != nil {
l.WithError(err).Warning("failed to start watching for configuration changes, no automatic update will be done")
}
if w != nil {
defer w.Close()
}
ws.Start()
<-ex
running = false
@ -92,8 +122,24 @@ func attemptStartBackend(g *gounicorn.GoUnicorn) {
if !running {
return
}
g.Kill()
log.WithField("logger", "authentik.router").Info("starting gunicorn")
err := g.Start()
log.WithField("logger", "authentik.router").WithError(err).Warning("gunicorn process died, restarting")
if err != nil {
log.WithField("logger", "authentik.router").WithError(err).Error("gunicorn failed to start, restarting")
continue
}
failedChecks := 0
for range time.Tick(30 * time.Second) {
if !g.IsRunning() {
log.WithField("logger", "authentik.router").Warningf("gunicorn process failed healthcheck %d times", failedChecks)
failedChecks += 1
}
if failedChecks >= 3 {
log.WithField("logger", "authentik.router").WithError(err).Error("gunicorn process failed healthcheck three times, restarting")
break
}
}
}
}

1
go.mod
View File

@ -42,6 +42,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
github.com/go-http-utils/fresh v0.0.0-20161124030543-7231e26a4b27 // indirect
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a // indirect

3
go.sum
View File

@ -66,6 +66,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/garyburd/redigo v1.6.4 h1:LFu2R3+ZOPgSMWMOL+saa/zXRjw0ID2G8FepO53BGlg=
github.com/garyburd/redigo v1.6.4/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw=
github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ=
@ -486,6 +488,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

View File

@ -5,46 +5,134 @@ import (
"fmt"
"net/url"
"os"
"path/filepath"
"reflect"
"strings"
"time"
env "github.com/Netflix/go-env"
"github.com/fsnotify/fsnotify"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
var cfg *Config
func getConfigPaths() []string {
configPaths := []string{"./authentik/lib/default.yml", "/etc/authentik/config.yml", ""}
globConfigPaths, _ := filepath.Glob("/etc/authentik/config.d/*.yml")
configPaths = append(configPaths, globConfigPaths...)
environment := "local"
if v, ok := os.LookupEnv("AUTHENTIK_ENV"); ok {
environment = v
}
computedConfigPaths := []string{}
for _, path := range configPaths {
path, err := filepath.Abs(path)
if err != nil {
continue
}
if stat, err := os.Stat(path); err == nil {
if !stat.IsDir() {
computedConfigPaths = append(computedConfigPaths, path)
} else {
envPaths := []string{
filepath.Join(path, environment+".yml"),
filepath.Join(path, environment+".env.yml"),
}
for _, envPath := range envPaths {
if stat, err = os.Stat(envPath); err == nil && !stat.IsDir() {
computedConfigPaths = append(computedConfigPaths, envPath)
}
}
}
}
}
return computedConfigPaths
}
func WatchChanges(callback func()) (*fsnotify.Watcher, error) {
configPaths := getConfigPaths()
w, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
// Start listening for events
go func() {
reload := false
for {
select {
// Read from errors
case err, ok := <-w.Errors:
if !ok {
// Channel was closed (i.e. Watcher.Close() was called)
return
}
log.WithError(err).Warning("failed to watch for file changes")
// Read from events
case ev, ok := <-w.Events:
if !ok {
// Channel was closed (i.e. Watcher.Close() was called)
return
}
// Ignores files we're not interested in
// We get the whole list again in case a file in a directory we're watching was created
// and that file is of interest
for _, f := range getConfigPaths() {
if f == ev.Name {
reload = true
log.Debugf("%s file change, setting configuration to be reloaded", ev.Name)
}
}
default:
if reload {
log.Infof("configuration files changed, reloading configuration")
Reload()
callback()
// Cooldown after configuration reloading
time.Sleep(14 * time.Second)
}
// Otherwise `select` fires all of the time
time.Sleep(1 * time.Second)
reload = false
}
}
}()
// Register the files to be watch
for _, p := range configPaths {
// Actually, watch the directory, not the files, as that's how fsnotify works
// We assume we're only getting valid files as getConfigPaths already handles all the necessary checks
err = w.Add(filepath.Dir(p))
log.Warningf("watching for changes for file %s in dir %s", p, filepath.Dir(p))
if err != nil {
return w, err
}
}
return w, nil
}
func Get() *Config {
if cfg == nil {
c := defaultConfig()
c.Setup("./authentik/lib/default.yml", "/etc/authentik/config.yml", "./local.env.yml")
c := &Config{}
c.Setup(getConfigPaths()...)
cfg = c
}
return cfg
}
func defaultConfig() *Config {
return &Config{
Debug: false,
Listen: ListenConfig{
HTTP: "0.0.0.0:9000",
HTTPS: "0.0.0.0:9443",
LDAP: "0.0.0.0:3389",
LDAPS: "0.0.0.0:6636",
Radius: "0.0.0.0:1812",
Metrics: "0.0.0.0:9300",
Debug: "0.0.0.0:9900",
},
Paths: PathsConfig{
Media: "./media",
},
LogLevel: "info",
ErrorReporting: ErrorReportingConfig{
Enabled: false,
SampleRate: 1,
},
}
func Reload() {
c := &Config{}
c.Setup(getConfigPaths()...)
cfg = c
}
func (c *Config) Setup(paths ...string) {

View File

@ -1,15 +1,20 @@
package gounicorn
import (
"fmt"
"net/http"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"syscall"
"time"
log "github.com/sirupsen/logrus"
"goauthentik.io/internal/config"
"goauthentik.io/internal/utils"
"goauthentik.io/internal/utils/web"
)
@ -18,6 +23,7 @@ type GoUnicorn struct {
log *log.Entry
p *exec.Cmd
pidFile *string
started bool
killed bool
alive bool
@ -27,6 +33,7 @@ func New() *GoUnicorn {
logger := log.WithField("logger", "authentik.router.unicorn")
g := &GoUnicorn{
log: logger,
pidFile: nil,
started: false,
killed: false,
alive: false,
@ -37,8 +44,13 @@ func New() *GoUnicorn {
}
func (g *GoUnicorn) initCmd() {
pidFile, _ := os.CreateTemp("", "authentik-gunicorn.*.pid")
g.pidFile = func() *string { s := pidFile.Name(); return &s }()
command := "gunicorn"
args := []string{"-c", "./lifecycle/gunicorn.conf.py", "authentik.root.asgi:application"}
if g.pidFile != nil {
args = append(args, "--pid", *g.pidFile)
}
if config.Get().Debug {
command = "./manage.py"
args = []string{"runserver"}
@ -55,16 +67,13 @@ func (g *GoUnicorn) IsRunning() bool {
}
func (g *GoUnicorn) Start() error {
if g.killed {
g.log.Debug("Not restarting gunicorn since we're shutdown")
return nil
}
if g.started {
g.initCmd()
}
g.killed = false
g.started = true
go g.healthcheck()
return g.p.Run()
return g.p.Start()
}
func (g *GoUnicorn) healthcheck() {
@ -96,8 +105,77 @@ func (g *GoUnicorn) healthcheck() {
}
}
func (g *GoUnicorn) Reload() {
g.log.WithField("method", "reload").Info("reloading gunicorn")
err := g.p.Process.Signal(syscall.SIGHUP)
if err != nil {
g.log.WithError(err).Warning("failed to reload gunicorn")
}
}
func (g *GoUnicorn) Restart() {
g.log.WithField("method", "restart").Info("restart gunicorn")
if g.pidFile == nil {
g.log.Warning("pidfile is non existent, cannot restart")
return
}
err := g.p.Process.Signal(syscall.SIGUSR2)
if err != nil {
g.log.WithError(err).Warning("failed to restart gunicorn")
return
}
newPidFile := fmt.Sprintf("%s.2", *g.pidFile)
// Wait for the new PID file to be created
for range time.Tick(1 * time.Second) {
_, err = os.Stat(newPidFile)
if err == nil || !os.IsNotExist(err) {
break
}
g.log.Debugf("waiting for new gunicorn pidfile to appear at %s", newPidFile)
}
if err != nil {
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
return
}
newPidB, err := os.ReadFile(newPidFile)
if err != nil {
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
return
}
newPidS := strings.TrimSpace(string(newPidB[:]))
newPid, err := strconv.Atoi(newPidS)
if err != nil {
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
return
}
g.log.Warningf("new gunicorn PID is %d", newPid)
newProcess, err := utils.FindProcess(newPid)
if newProcess == nil || err != nil {
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
return
}
// The new process has started, let's gracefully kill the old one
g.log.Warningf("killing old gunicorn")
err = g.p.Process.Signal(syscall.SIGTERM)
if err != nil {
g.log.Warning("failed to kill old instance of gunicorn")
}
g.p.Process = newProcess
// No need to close any files and the .2 pid file is deleted by Gunicorn
}
func (g *GoUnicorn) Kill() {
g.killed = true
if !g.started {
return
}
var err error
if runtime.GOOS == "darwin" {
g.log.WithField("method", "kill").Warning("stopping gunicorn")
@ -109,4 +187,8 @@ func (g *GoUnicorn) Kill() {
if err != nil {
g.log.WithError(err).Warning("failed to stop gunicorn")
}
if g.pidFile != nil {
os.Remove(*g.pidFile)
}
g.killed = true
}

39
internal/utils/process.go Normal file
View File

@ -0,0 +1,39 @@
package utils
import (
"errors"
"fmt"
"os"
"syscall"
)
func FindProcess(pid int) (*os.Process, error) {
if pid <= 0 {
return nil, fmt.Errorf("invalid pid %v", pid)
}
// The error doesn't mean anything on Unix systems, let's just check manually
// that the new gunicorn master has properly started
// https://github.com/golang/go/issues/34396
proc, err := os.FindProcess(int(pid))
if err != nil {
return nil, err
}
err = proc.Signal(syscall.Signal(0))
if err == nil {
return proc, nil
}
if errors.Is(err, os.ErrProcessDone) {
return nil, nil
}
errno, ok := err.(syscall.Errno)
if !ok {
return nil, err
}
switch errno {
case syscall.ESRCH:
return nil, nil
case syscall.EPERM:
return proc, nil
}
return nil, err
}