Compare commits
7 Commits
trustchain
...
server-han
Author | SHA1 | Date |
---|---|---|
Jens Langhammer | 241059f56b | |
Jens Langhammer | 18db59ad2d | |
Marc 'risson' Schmitt | d3ef158360 | |
Marc 'risson' Schmitt | 969ec188a7 | |
Marc 'risson' Schmitt | 2a15004232 | |
Marc 'risson' Schmitt | 4e1d8543e3 | |
Marc 'risson' Schmitt | fc5f6d6677 |
|
@ -11,7 +11,11 @@ postgresql:
|
|||
listen:
|
||||
listen_http: 0.0.0.0:9000
|
||||
listen_https: 0.0.0.0:9443
|
||||
listen_ldap: 0.0.0.0:3389
|
||||
listen_ldaps: 0.0.0.0:6636
|
||||
listen_radius: 0.0.0.0:1812
|
||||
listen_metrics: 0.0.0.0:9300
|
||||
listen_debug: 0.0.0.0:9900
|
||||
|
||||
redis:
|
||||
host: localhost
|
||||
|
@ -25,6 +29,9 @@ redis:
|
|||
cache_timeout_policies: 300
|
||||
cache_timeout_reputation: 300
|
||||
|
||||
paths:
|
||||
media: ./media
|
||||
|
||||
debug: false
|
||||
|
||||
log_level: info
|
||||
|
|
|
@ -4,11 +4,15 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"goauthentik.io/internal/common"
|
||||
"goauthentik.io/internal/config"
|
||||
"goauthentik.io/internal/constants"
|
||||
|
@ -70,6 +74,21 @@ var rootCmd = &cobra.Command{
|
|||
l.Info("shutting down gunicorn")
|
||||
g.Kill()
|
||||
}()
|
||||
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, syscall.SIGHUP, syscall.SIGUSR2)
|
||||
go func() {
|
||||
sig := <-c
|
||||
if sig == syscall.SIGHUP {
|
||||
log.Info("SIGHUP received, forwarding to gunicorn")
|
||||
g.Reload()
|
||||
}
|
||||
if sig == syscall.SIGUSR2 {
|
||||
log.Info("SIGUSR2 received, restarting gunicorn")
|
||||
g.Restart()
|
||||
}
|
||||
}()
|
||||
|
||||
ws := web.NewWebServer(g)
|
||||
g.HealthyCallback = func() {
|
||||
if !config.Get().Outposts.DisableEmbeddedOutpost {
|
||||
|
@ -78,6 +97,17 @@ var rootCmd = &cobra.Command{
|
|||
}
|
||||
go web.RunMetricsServer()
|
||||
go attemptStartBackend(g)
|
||||
|
||||
w, err := config.WatchChanges(func() {
|
||||
g.Restart()
|
||||
})
|
||||
if err != nil {
|
||||
l.WithError(err).Warning("failed to start watching for configuration changes, no automatic update will be done")
|
||||
}
|
||||
if w != nil {
|
||||
defer w.Close()
|
||||
}
|
||||
|
||||
ws.Start()
|
||||
<-ex
|
||||
running = false
|
||||
|
@ -92,8 +122,24 @@ func attemptStartBackend(g *gounicorn.GoUnicorn) {
|
|||
if !running {
|
||||
return
|
||||
}
|
||||
g.Kill()
|
||||
log.WithField("logger", "authentik.router").Info("starting gunicorn")
|
||||
err := g.Start()
|
||||
log.WithField("logger", "authentik.router").WithError(err).Warning("gunicorn process died, restarting")
|
||||
if err != nil {
|
||||
log.WithField("logger", "authentik.router").WithError(err).Error("gunicorn failed to start, restarting")
|
||||
continue
|
||||
}
|
||||
failedChecks := 0
|
||||
for range time.Tick(30 * time.Second) {
|
||||
if !g.IsRunning() {
|
||||
log.WithField("logger", "authentik.router").Warningf("gunicorn process failed healthcheck %d times", failedChecks)
|
||||
failedChecks += 1
|
||||
}
|
||||
if failedChecks >= 3 {
|
||||
log.WithField("logger", "authentik.router").WithError(err).Error("gunicorn process failed healthcheck three times, restarting")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
1
go.mod
1
go.mod
|
@ -42,6 +42,7 @@ require (
|
|||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.1 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
|
||||
github.com/go-http-utils/fresh v0.0.0-20161124030543-7231e26a4b27 // indirect
|
||||
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a // indirect
|
||||
|
|
3
go.sum
3
go.sum
|
@ -66,6 +66,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
|
|||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
|
||||
github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/garyburd/redigo v1.6.4 h1:LFu2R3+ZOPgSMWMOL+saa/zXRjw0ID2G8FepO53BGlg=
|
||||
github.com/garyburd/redigo v1.6.4/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw=
|
||||
github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ=
|
||||
|
@ -486,6 +488,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
|
@ -5,46 +5,134 @@ import (
|
|||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
env "github.com/Netflix/go-env"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var cfg *Config
|
||||
|
||||
func getConfigPaths() []string {
|
||||
configPaths := []string{"./authentik/lib/default.yml", "/etc/authentik/config.yml", ""}
|
||||
globConfigPaths, _ := filepath.Glob("/etc/authentik/config.d/*.yml")
|
||||
configPaths = append(configPaths, globConfigPaths...)
|
||||
|
||||
environment := "local"
|
||||
if v, ok := os.LookupEnv("AUTHENTIK_ENV"); ok {
|
||||
environment = v
|
||||
}
|
||||
|
||||
computedConfigPaths := []string{}
|
||||
|
||||
for _, path := range configPaths {
|
||||
path, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if stat, err := os.Stat(path); err == nil {
|
||||
if !stat.IsDir() {
|
||||
computedConfigPaths = append(computedConfigPaths, path)
|
||||
} else {
|
||||
envPaths := []string{
|
||||
filepath.Join(path, environment+".yml"),
|
||||
filepath.Join(path, environment+".env.yml"),
|
||||
}
|
||||
for _, envPath := range envPaths {
|
||||
if stat, err = os.Stat(envPath); err == nil && !stat.IsDir() {
|
||||
computedConfigPaths = append(computedConfigPaths, envPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return computedConfigPaths
|
||||
}
|
||||
|
||||
func WatchChanges(callback func()) (*fsnotify.Watcher, error) {
|
||||
configPaths := getConfigPaths()
|
||||
|
||||
w, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Start listening for events
|
||||
go func() {
|
||||
reload := false
|
||||
for {
|
||||
select {
|
||||
// Read from errors
|
||||
case err, ok := <-w.Errors:
|
||||
if !ok {
|
||||
// Channel was closed (i.e. Watcher.Close() was called)
|
||||
return
|
||||
}
|
||||
log.WithError(err).Warning("failed to watch for file changes")
|
||||
// Read from events
|
||||
case ev, ok := <-w.Events:
|
||||
if !ok {
|
||||
// Channel was closed (i.e. Watcher.Close() was called)
|
||||
return
|
||||
}
|
||||
|
||||
// Ignores files we're not interested in
|
||||
// We get the whole list again in case a file in a directory we're watching was created
|
||||
// and that file is of interest
|
||||
for _, f := range getConfigPaths() {
|
||||
if f == ev.Name {
|
||||
reload = true
|
||||
log.Debugf("%s file change, setting configuration to be reloaded", ev.Name)
|
||||
}
|
||||
}
|
||||
default:
|
||||
if reload {
|
||||
log.Infof("configuration files changed, reloading configuration")
|
||||
Reload()
|
||||
callback()
|
||||
// Cooldown after configuration reloading
|
||||
time.Sleep(14 * time.Second)
|
||||
}
|
||||
// Otherwise `select` fires all of the time
|
||||
time.Sleep(1 * time.Second)
|
||||
reload = false
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Register the files to be watch
|
||||
for _, p := range configPaths {
|
||||
// Actually, watch the directory, not the files, as that's how fsnotify works
|
||||
// We assume we're only getting valid files as getConfigPaths already handles all the necessary checks
|
||||
err = w.Add(filepath.Dir(p))
|
||||
log.Warningf("watching for changes for file %s in dir %s", p, filepath.Dir(p))
|
||||
if err != nil {
|
||||
return w, err
|
||||
}
|
||||
}
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
func Get() *Config {
|
||||
if cfg == nil {
|
||||
c := defaultConfig()
|
||||
c.Setup("./authentik/lib/default.yml", "/etc/authentik/config.yml", "./local.env.yml")
|
||||
c := &Config{}
|
||||
c.Setup(getConfigPaths()...)
|
||||
cfg = c
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func defaultConfig() *Config {
|
||||
return &Config{
|
||||
Debug: false,
|
||||
Listen: ListenConfig{
|
||||
HTTP: "0.0.0.0:9000",
|
||||
HTTPS: "0.0.0.0:9443",
|
||||
LDAP: "0.0.0.0:3389",
|
||||
LDAPS: "0.0.0.0:6636",
|
||||
Radius: "0.0.0.0:1812",
|
||||
Metrics: "0.0.0.0:9300",
|
||||
Debug: "0.0.0.0:9900",
|
||||
},
|
||||
Paths: PathsConfig{
|
||||
Media: "./media",
|
||||
},
|
||||
LogLevel: "info",
|
||||
ErrorReporting: ErrorReportingConfig{
|
||||
Enabled: false,
|
||||
SampleRate: 1,
|
||||
},
|
||||
}
|
||||
func Reload() {
|
||||
c := &Config{}
|
||||
c.Setup(getConfigPaths()...)
|
||||
cfg = c
|
||||
}
|
||||
|
||||
func (c *Config) Setup(paths ...string) {
|
||||
|
|
|
@ -1,15 +1,20 @@
|
|||
package gounicorn
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"goauthentik.io/internal/config"
|
||||
"goauthentik.io/internal/utils"
|
||||
"goauthentik.io/internal/utils/web"
|
||||
)
|
||||
|
||||
|
@ -18,6 +23,7 @@ type GoUnicorn struct {
|
|||
|
||||
log *log.Entry
|
||||
p *exec.Cmd
|
||||
pidFile *string
|
||||
started bool
|
||||
killed bool
|
||||
alive bool
|
||||
|
@ -27,6 +33,7 @@ func New() *GoUnicorn {
|
|||
logger := log.WithField("logger", "authentik.router.unicorn")
|
||||
g := &GoUnicorn{
|
||||
log: logger,
|
||||
pidFile: nil,
|
||||
started: false,
|
||||
killed: false,
|
||||
alive: false,
|
||||
|
@ -37,8 +44,13 @@ func New() *GoUnicorn {
|
|||
}
|
||||
|
||||
func (g *GoUnicorn) initCmd() {
|
||||
pidFile, _ := os.CreateTemp("", "authentik-gunicorn.*.pid")
|
||||
g.pidFile = func() *string { s := pidFile.Name(); return &s }()
|
||||
command := "gunicorn"
|
||||
args := []string{"-c", "./lifecycle/gunicorn.conf.py", "authentik.root.asgi:application"}
|
||||
if g.pidFile != nil {
|
||||
args = append(args, "--pid", *g.pidFile)
|
||||
}
|
||||
if config.Get().Debug {
|
||||
command = "./manage.py"
|
||||
args = []string{"runserver"}
|
||||
|
@ -55,16 +67,13 @@ func (g *GoUnicorn) IsRunning() bool {
|
|||
}
|
||||
|
||||
func (g *GoUnicorn) Start() error {
|
||||
if g.killed {
|
||||
g.log.Debug("Not restarting gunicorn since we're shutdown")
|
||||
return nil
|
||||
}
|
||||
if g.started {
|
||||
g.initCmd()
|
||||
}
|
||||
g.killed = false
|
||||
g.started = true
|
||||
go g.healthcheck()
|
||||
return g.p.Run()
|
||||
return g.p.Start()
|
||||
}
|
||||
|
||||
func (g *GoUnicorn) healthcheck() {
|
||||
|
@ -96,8 +105,77 @@ func (g *GoUnicorn) healthcheck() {
|
|||
}
|
||||
}
|
||||
|
||||
func (g *GoUnicorn) Reload() {
|
||||
g.log.WithField("method", "reload").Info("reloading gunicorn")
|
||||
err := g.p.Process.Signal(syscall.SIGHUP)
|
||||
if err != nil {
|
||||
g.log.WithError(err).Warning("failed to reload gunicorn")
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GoUnicorn) Restart() {
|
||||
g.log.WithField("method", "restart").Info("restart gunicorn")
|
||||
if g.pidFile == nil {
|
||||
g.log.Warning("pidfile is non existent, cannot restart")
|
||||
return
|
||||
}
|
||||
|
||||
err := g.p.Process.Signal(syscall.SIGUSR2)
|
||||
if err != nil {
|
||||
g.log.WithError(err).Warning("failed to restart gunicorn")
|
||||
return
|
||||
}
|
||||
|
||||
newPidFile := fmt.Sprintf("%s.2", *g.pidFile)
|
||||
|
||||
// Wait for the new PID file to be created
|
||||
for range time.Tick(1 * time.Second) {
|
||||
_, err = os.Stat(newPidFile)
|
||||
if err == nil || !os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
g.log.Debugf("waiting for new gunicorn pidfile to appear at %s", newPidFile)
|
||||
}
|
||||
if err != nil {
|
||||
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
|
||||
return
|
||||
}
|
||||
|
||||
newPidB, err := os.ReadFile(newPidFile)
|
||||
if err != nil {
|
||||
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
|
||||
return
|
||||
}
|
||||
newPidS := strings.TrimSpace(string(newPidB[:]))
|
||||
newPid, err := strconv.Atoi(newPidS)
|
||||
if err != nil {
|
||||
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
|
||||
return
|
||||
}
|
||||
g.log.Warningf("new gunicorn PID is %d", newPid)
|
||||
|
||||
newProcess, err := utils.FindProcess(newPid)
|
||||
if newProcess == nil || err != nil {
|
||||
g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
|
||||
return
|
||||
}
|
||||
|
||||
// The new process has started, let's gracefully kill the old one
|
||||
g.log.Warningf("killing old gunicorn")
|
||||
err = g.p.Process.Signal(syscall.SIGTERM)
|
||||
if err != nil {
|
||||
g.log.Warning("failed to kill old instance of gunicorn")
|
||||
}
|
||||
|
||||
g.p.Process = newProcess
|
||||
|
||||
// No need to close any files and the .2 pid file is deleted by Gunicorn
|
||||
}
|
||||
|
||||
func (g *GoUnicorn) Kill() {
|
||||
g.killed = true
|
||||
if !g.started {
|
||||
return
|
||||
}
|
||||
var err error
|
||||
if runtime.GOOS == "darwin" {
|
||||
g.log.WithField("method", "kill").Warning("stopping gunicorn")
|
||||
|
@ -109,4 +187,8 @@ func (g *GoUnicorn) Kill() {
|
|||
if err != nil {
|
||||
g.log.WithError(err).Warning("failed to stop gunicorn")
|
||||
}
|
||||
if g.pidFile != nil {
|
||||
os.Remove(*g.pidFile)
|
||||
}
|
||||
g.killed = true
|
||||
}
|
||||
|
|
|
@ -0,0 +1,39 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func FindProcess(pid int) (*os.Process, error) {
|
||||
if pid <= 0 {
|
||||
return nil, fmt.Errorf("invalid pid %v", pid)
|
||||
}
|
||||
// The error doesn't mean anything on Unix systems, let's just check manually
|
||||
// that the new gunicorn master has properly started
|
||||
// https://github.com/golang/go/issues/34396
|
||||
proc, err := os.FindProcess(int(pid))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = proc.Signal(syscall.Signal(0))
|
||||
if err == nil {
|
||||
return proc, nil
|
||||
}
|
||||
if errors.Is(err, os.ErrProcessDone) {
|
||||
return nil, nil
|
||||
}
|
||||
errno, ok := err.(syscall.Errno)
|
||||
if !ok {
|
||||
return nil, err
|
||||
}
|
||||
switch errno {
|
||||
case syscall.ESRCH:
|
||||
return nil, nil
|
||||
case syscall.EPERM:
|
||||
return proc, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
Reference in New Issue