From 7158c9d2eae9b99f7d603db8083254e9df0e7bf0 Mon Sep 17 00:00:00 2001 From: Jens L Date: Thu, 9 Sep 2021 15:52:24 +0200 Subject: [PATCH] core: metrics v2 (#1370) * outposts: add ldap metrics, move ping to 9100 Signed-off-by: Jens Langhammer * outpost: add flow_executor metrics Signed-off-by: Jens Langhammer * use port 9300 for metrics, add core metrics port Signed-off-by: Jens Langhammer * outposts/controllers/k8s: add service monitor creation support Signed-off-by: Jens Langhammer --- authentik/lib/default.yml | 1 + .../controllers/k8s/service_monitor.py | 150 ++++++++++++++++++ authentik/outposts/controllers/kubernetes.py | 4 +- .../providers/ldap/controllers/docker.py | 1 + .../providers/ldap/controllers/kubernetes.py | 1 + .../providers/proxy/controllers/docker.py | 2 +- .../providers/proxy/controllers/kubernetes.py | 2 +- cmd/server/main.go | 1 + internal/config/struct.go | 1 + internal/outpost/flow.go | 31 +++- internal/outpost/ldap/api.go | 17 +- internal/outpost/ldap/bind.go | 20 ++- internal/outpost/ldap/instance_bind.go | 32 ++++ internal/outpost/ldap/instance_search.go | 39 +++++ internal/outpost/ldap/metrics/metrics.go | 33 ++++ internal/outpost/ldap/search.go | 8 + internal/outpost/proxyv2/metrics/metrics.go | 5 +- internal/outpost/proxyv2/proxyv2.go | 1 - internal/web/metrics.go | 58 +++++++ internal/web/{web_proxy.go => proxy.go} | 13 ++ internal/web/{web_ssl.go => ssl.go} | 0 internal/web/{web_static.go => static.go} | 0 .../{web_static_utils.go => static_utils.go} | 0 internal/web/{web_utils.go => utils.go} | 0 ldap.Dockerfile | 2 +- proxy.Dockerfile | 2 +- website/docs/outposts/outposts.md | 1 + 27 files changed, 400 insertions(+), 25 deletions(-) create mode 100644 authentik/outposts/controllers/k8s/service_monitor.py create mode 100644 internal/outpost/ldap/metrics/metrics.go create mode 100644 internal/web/metrics.go rename internal/web/{web_proxy.go => proxy.go} (78%) rename internal/web/{web_ssl.go => ssl.go} (100%) rename internal/web/{web_static.go => static.go} (100%) rename internal/web/{web_static_utils.go => static_utils.go} (100%) rename internal/web/{web_utils.go => utils.go} (100%) diff --git a/authentik/lib/default.yml b/authentik/lib/default.yml index 0a1861dc9..92a1f0a01 100644 --- a/authentik/lib/default.yml +++ b/authentik/lib/default.yml @@ -9,6 +9,7 @@ postgresql: web: listen: 0.0.0.0:9000 listen_tls: 0.0.0.0:9443 + listen_metrics: 0.0.0.0:9100 load_local_files: false outpost_port_offset: 0 diff --git a/authentik/outposts/controllers/k8s/service_monitor.py b/authentik/outposts/controllers/k8s/service_monitor.py new file mode 100644 index 000000000..2fadbc6dd --- /dev/null +++ b/authentik/outposts/controllers/k8s/service_monitor.py @@ -0,0 +1,150 @@ +"""Kubernetes Prometheus ServiceMonitor Reconciler""" +from dataclasses import asdict, dataclass, field +from typing import TYPE_CHECKING + +from dacite import from_dict +from kubernetes.client import ApiextensionsV1Api, CustomObjectsApi + +from authentik.outposts.controllers.base import FIELD_MANAGER +from authentik.outposts.controllers.k8s.base import KubernetesObjectReconciler + +if TYPE_CHECKING: + from authentik.outposts.controllers.kubernetes import KubernetesController + + +@dataclass +class PrometheusServiceMonitorSpecEndpoint: + """Prometheus ServiceMonitor endpoint spec""" + + port: str + path: str = field(default="/metrics") + + +@dataclass +class PrometheusServiceMonitorSpecSelector: + """Prometheus ServiceMonitor selector spec""" + + # pylint: disable=invalid-name + matchLabels: dict + + +@dataclass +class PrometheusServiceMonitorSpec: + """Prometheus ServiceMonitor spec""" + + endpoints: list[PrometheusServiceMonitorSpecEndpoint] + # pylint: disable=invalid-name + selector: PrometheusServiceMonitorSpecSelector + + +@dataclass +class PrometheusServiceMonitorMetadata: + """Prometheus ServiceMonitor metadata""" + + name: str + namespace: str + labels: dict = field(default_factory=dict) + + +@dataclass +class PrometheusServiceMonitor: + """Prometheus ServiceMonitor""" + + # pylint: disable=invalid-name + apiVersion: str + kind: str + metadata: PrometheusServiceMonitorMetadata + spec: PrometheusServiceMonitorSpec + + +CRD_NAME = "servicemonitors.monitoring.coreos.com" +CRD_GROUP = "monitoring.coreos.com" +CRD_VERSION = "v1" +CRD_PLURAL = "servicemonitors" + + +class PrometheusServiceMonitorReconciler(KubernetesObjectReconciler[PrometheusServiceMonitor]): + """Kubernetes Prometheus ServiceMonitor Reconciler""" + + def __init__(self, controller: "KubernetesController") -> None: + super().__init__(controller) + self.api_ex = ApiextensionsV1Api(controller.client) + self.api = CustomObjectsApi(controller.client) + + @property + def noop(self) -> bool: + return not self._crd_exists() + + def _crd_exists(self) -> bool: + """Check if the Prometheus ServiceMonitor exists""" + return bool( + len( + self.api_ex.list_custom_resource_definition( + field_selector=f"metadata.name={CRD_NAME}" + ).items + ) + ) + + def get_reference_object(self) -> PrometheusServiceMonitor: + """Get service monitor object for outpost""" + return PrometheusServiceMonitor( + apiVersion=f"{CRD_GROUP}/{CRD_VERSION}", + kind="ServiceMonitor", + metadata=PrometheusServiceMonitorMetadata( + name=self.name, + namespace=self.namespace, + labels=self.get_object_meta().labels, + ), + spec=PrometheusServiceMonitorSpec( + endpoints=[ + PrometheusServiceMonitorSpecEndpoint( + port="http-metrics", + ) + ], + selector=PrometheusServiceMonitorSpecSelector( + matchLabels=self.get_object_meta(name=self.name).labels, + ), + ), + ) + + def create(self, reference: PrometheusServiceMonitor): + return self.api.create_namespaced_custom_object( + group=CRD_GROUP, + version=CRD_VERSION, + plural=CRD_PLURAL, + namespace=self.namespace, + body=asdict(reference), + field_manager=FIELD_MANAGER, + ) + + def delete(self, reference: PrometheusServiceMonitor): + return self.api.delete_namespaced_custom_object( + group=CRD_GROUP, + version=CRD_VERSION, + namespace=self.namespace, + plural=CRD_PLURAL, + name=self.name, + ) + + def retrieve(self) -> PrometheusServiceMonitor: + return from_dict( + PrometheusServiceMonitor, + self.api.get_namespaced_custom_object( + group=CRD_GROUP, + version=CRD_VERSION, + namespace=self.namespace, + plural=CRD_PLURAL, + name=self.name, + ), + ) + + def update(self, current: PrometheusServiceMonitor, reference: PrometheusServiceMonitor): + return self.api.patch_namespaced_custom_object( + group=CRD_GROUP, + version=CRD_VERSION, + namespace=self.namespace, + plural=CRD_PLURAL, + name=self.name, + body=asdict(reference), + field_manager=FIELD_MANAGER, + ) diff --git a/authentik/outposts/controllers/kubernetes.py b/authentik/outposts/controllers/kubernetes.py index 0285d5c4a..8fd6a1b57 100644 --- a/authentik/outposts/controllers/kubernetes.py +++ b/authentik/outposts/controllers/kubernetes.py @@ -13,6 +13,7 @@ from authentik.outposts.controllers.k8s.base import KubernetesObjectReconciler from authentik.outposts.controllers.k8s.deployment import DeploymentReconciler from authentik.outposts.controllers.k8s.secret import SecretReconciler from authentik.outposts.controllers.k8s.service import ServiceReconciler +from authentik.outposts.controllers.k8s.service_monitor import PrometheusServiceMonitorReconciler from authentik.outposts.models import KubernetesServiceConnection, Outpost, ServiceConnectionInvalid @@ -32,8 +33,9 @@ class KubernetesController(BaseController): "secret": SecretReconciler, "deployment": DeploymentReconciler, "service": ServiceReconciler, + "prometheus servicemonitor": PrometheusServiceMonitorReconciler, } - self.reconcile_order = ["secret", "deployment", "service"] + self.reconcile_order = ["secret", "deployment", "service", "prometheus servicemonitor"] def up(self): try: diff --git a/authentik/providers/ldap/controllers/docker.py b/authentik/providers/ldap/controllers/docker.py index 27d972917..95395cecd 100644 --- a/authentik/providers/ldap/controllers/docker.py +++ b/authentik/providers/ldap/controllers/docker.py @@ -12,4 +12,5 @@ class LDAPDockerController(DockerController): self.deployment_ports = [ DeploymentPort(389, "ldap", "tcp", 3389), DeploymentPort(636, "ldaps", "tcp", 6636), + DeploymentPort(9300, "http-metrics", "tcp", 9300), ] diff --git a/authentik/providers/ldap/controllers/kubernetes.py b/authentik/providers/ldap/controllers/kubernetes.py index e267979db..b7449213d 100644 --- a/authentik/providers/ldap/controllers/kubernetes.py +++ b/authentik/providers/ldap/controllers/kubernetes.py @@ -12,4 +12,5 @@ class LDAPKubernetesController(KubernetesController): self.deployment_ports = [ DeploymentPort(389, "ldap", "tcp", 3389), DeploymentPort(636, "ldaps", "tcp", 6636), + DeploymentPort(9300, "http-metrics", "tcp", 9300), ] diff --git a/authentik/providers/proxy/controllers/docker.py b/authentik/providers/proxy/controllers/docker.py index 9727f874a..dfed9ff09 100644 --- a/authentik/providers/proxy/controllers/docker.py +++ b/authentik/providers/proxy/controllers/docker.py @@ -14,7 +14,7 @@ class ProxyDockerController(DockerController): super().__init__(outpost, connection) self.deployment_ports = [ DeploymentPort(9000, "http", "tcp"), - DeploymentPort(9100, "http-metrics", "tcp"), + DeploymentPort(9300, "http-metrics", "tcp"), DeploymentPort(9443, "https", "tcp"), ] diff --git a/authentik/providers/proxy/controllers/kubernetes.py b/authentik/providers/proxy/controllers/kubernetes.py index d3a5bd6ec..f8ca3d1ef 100644 --- a/authentik/providers/proxy/controllers/kubernetes.py +++ b/authentik/providers/proxy/controllers/kubernetes.py @@ -13,7 +13,7 @@ class ProxyKubernetesController(KubernetesController): super().__init__(outpost, connection) self.deployment_ports = [ DeploymentPort(9000, "http", "tcp"), - DeploymentPort(9100, "http-metrics", "tcp"), + DeploymentPort(9300, "http-metrics", "tcp"), DeploymentPort(9443, "https", "tcp"), ] self.reconcilers["ingress"] = IngressReconciler diff --git a/cmd/server/main.go b/cmd/server/main.go index c278598bc..e93ab1c93 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -57,6 +57,7 @@ func main() { ws := web.NewWebServer() defer g.Kill() defer ws.Shutdown() + go web.RunMetricsServer() for { go attemptStartBackend(g) ws.Start() diff --git a/internal/config/struct.go b/internal/config/struct.go index b28bb7df4..242483c72 100644 --- a/internal/config/struct.go +++ b/internal/config/struct.go @@ -29,6 +29,7 @@ type RedisConfig struct { type WebConfig struct { Listen string `yaml:"listen"` ListenTLS string `yaml:"listen_tls"` + ListenMetrics string `yaml:"listen_metrics"` LoadLocalFiles bool `yaml:"load_local_files" env:"AUTHENTIK_WEB_LOAD_LOCAL_FILES"` DisableEmbeddedOutpost bool `yaml:"disable_embedded_outpost" env:"AUTHENTIK_WEB__DISABLE_EMBEDDED_OUTPOST"` } diff --git a/internal/outpost/flow.go b/internal/outpost/flow.go index 591776166..bc8e8e904 100644 --- a/internal/outpost/flow.go +++ b/internal/outpost/flow.go @@ -12,6 +12,8 @@ import ( "strings" "github.com/getsentry/sentry-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" log "github.com/sirupsen/logrus" "goauthentik.io/api" "goauthentik.io/internal/constants" @@ -21,6 +23,17 @@ import ( type StageComponent string +var ( + FlowTimingGet = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "authentik_outpost_flow_timing_get", + Help: "Duration it took to get a challenge", + }, []string{"stage", "flow", "client", "user"}) + FlowTimingPost = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "authentik_outpost_flow_timing_post", + Help: "Duration it took to send a challenge", + }, []string{"stage", "flow", "client", "user"}) +) + const ( StageIdentification = StageComponent("ak-stage-identification") StagePassword = StageComponent("ak-stage-password") @@ -38,6 +51,7 @@ type FlowExecutor struct { Answers map[StageComponent]string Context context.Context + cip string api *api.APIClient flowSlug string log *log.Entry @@ -75,6 +89,7 @@ func NewFlowExecutor(ctx context.Context, flowSlug string, refConfig *api.Config log: l, token: token, sp: rsp, + cip: "", } } @@ -89,7 +104,8 @@ type ChallengeInt interface { } func (fe *FlowExecutor) DelegateClientIP(a net.Addr) { - fe.api.GetConfig().AddDefaultHeader(HeaderAuthentikRemoteIP, utils.GetIP(a)) + fe.cip = utils.GetIP(a) + fe.api.GetConfig().AddDefaultHeader(HeaderAuthentikRemoteIP, fe.cip) } func (fe *FlowExecutor) CheckApplicationAccess(appSlug string) (bool, error) { @@ -142,6 +158,12 @@ func (fe *FlowExecutor) solveFlowChallenge(depth int) (bool, error) { gcsp.SetTag("ak_challenge", string(ch.GetType())) gcsp.SetTag("ak_component", ch.GetComponent()) gcsp.Finish() + FlowTimingGet.With(prometheus.Labels{ + "stage": ch.GetComponent(), + "flow": fe.flowSlug, + "client": fe.cip, + "user": fe.Answers[StageIdentification], + }).Observe(float64(gcsp.EndTime.Sub(gcsp.StartTime))) // Resole challenge scsp := sentry.StartSpan(fe.Context, "authentik.outposts.flow_executor.solve_challenge") @@ -202,6 +224,13 @@ func (fe *FlowExecutor) solveFlowChallenge(depth int) (bool, error) { } } } + FlowTimingPost.With(prometheus.Labels{ + "stage": ch.GetComponent(), + "flow": fe.flowSlug, + "client": fe.cip, + "user": fe.Answers[StageIdentification], + }).Observe(float64(scsp.EndTime.Sub(scsp.StartTime))) + if depth >= 10 { return false, errors.New("exceeded stage recursion depth") } diff --git a/internal/outpost/ldap/api.go b/internal/outpost/ldap/api.go index eb63b2263..7f6e84d99 100644 --- a/internal/outpost/ldap/api.go +++ b/internal/outpost/ldap/api.go @@ -6,13 +6,13 @@ import ( "errors" "fmt" "net" - "net/http" "strings" "sync" "github.com/go-openapi/strfmt" "github.com/pires/go-proxyproto" log "github.com/sirupsen/logrus" + "goauthentik.io/internal/outpost/ldap/metrics" ) const ( @@ -65,16 +65,6 @@ func (ls *LDAPServer) Refresh() error { return nil } -func (ls *LDAPServer) StartHTTPServer() error { - listen := "0.0.0.0:9000" // same port as proxy - m := http.NewServeMux() - m.HandleFunc("/akprox/ping", func(rw http.ResponseWriter, r *http.Request) { - rw.WriteHeader(204) - }) - ls.log.WithField("listen", listen).Info("Starting http server") - return http.ListenAndServe(listen, m) -} - func (ls *LDAPServer) StartLDAPServer() error { listen := "0.0.0.0:3389" @@ -126,10 +116,7 @@ func (ls *LDAPServer) Start() error { wg.Add(3) go func() { defer wg.Done() - err := ls.StartHTTPServer() - if err != nil { - panic(err) - } + metrics.RunServer() }() go func() { defer wg.Done() diff --git a/internal/outpost/ldap/bind.go b/internal/outpost/ldap/bind.go index 303fd5de3..c2dfb7e4d 100644 --- a/internal/outpost/ldap/bind.go +++ b/internal/outpost/ldap/bind.go @@ -8,7 +8,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/google/uuid" "github.com/nmcclain/ldap" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "goauthentik.io/internal/outpost/ldap/metrics" "goauthentik.io/internal/utils" ) @@ -27,7 +29,6 @@ func (ls *LDAPServer) Bind(bindDN string, bindPW string, conn net.Conn) (ldap.LD rid := uuid.New().String() span.SetTag("request_uid", rid) span.SetTag("user.username", bindDN) - defer span.Finish() bindDN = strings.ToLower(bindDN) req := BindRequest{ @@ -38,7 +39,16 @@ func (ls *LDAPServer) Bind(bindDN string, bindPW string, conn net.Conn) (ldap.LD id: rid, ctx: span.Context(), } - req.log.Info("Bind request") + defer func() { + span.Finish() + metrics.Requests.With(prometheus.Labels{ + "type": "bind", + "filter": "", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Observe(float64(span.EndTime.Sub(span.StartTime))) + req.log.WithField("took-ms", span.EndTime.Sub(span.StartTime).Milliseconds()).Info("Bind request") + }() for _, instance := range ls.providers { username, err := instance.getUsername(bindDN) if err == nil { @@ -48,6 +58,12 @@ func (ls *LDAPServer) Bind(bindDN string, bindPW string, conn net.Conn) (ldap.LD } } req.log.WithField("request", "bind").Warning("No provider found for request") + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "bind", + "reason": "no_provider", + "dn": bindDN, + "client": utils.GetIP(conn.RemoteAddr()), + }).Inc() return ldap.LDAPResultOperationsError, nil } diff --git a/internal/outpost/ldap/instance_bind.go b/internal/outpost/ldap/instance_bind.go index 6f1d05a67..846271922 100644 --- a/internal/outpost/ldap/instance_bind.go +++ b/internal/outpost/ldap/instance_bind.go @@ -8,9 +8,11 @@ import ( "github.com/getsentry/sentry-go" goldap "github.com/go-ldap/ldap/v3" "github.com/nmcclain/ldap" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "goauthentik.io/api" "goauthentik.io/internal/outpost" + "goauthentik.io/internal/outpost/ldap/metrics" "goauthentik.io/internal/utils" ) @@ -48,9 +50,21 @@ func (pi *ProviderInstance) Bind(username string, req BindRequest) (ldap.LDAPRes passed, err := fe.Execute() if !passed { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "bind", + "reason": "invalid_credentials", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.LDAPResultInvalidCredentials, nil } if err != nil { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "bind", + "reason": "flow_error", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() req.log.WithError(err).Warning("failed to execute flow") return ldap.LDAPResultOperationsError, nil } @@ -58,9 +72,21 @@ func (pi *ProviderInstance) Bind(username string, req BindRequest) (ldap.LDAPRes access, err := fe.CheckApplicationAccess(pi.appSlug) if !access { req.log.Info("Access denied for user") + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "bind", + "reason": "access_denied", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.LDAPResultInsufficientAccessRights, nil } if err != nil { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "bind", + "reason": "access_check_fail", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() req.log.WithError(err).Warning("failed to check access") return ldap.LDAPResultOperationsError, nil } @@ -69,6 +95,12 @@ func (pi *ProviderInstance) Bind(username string, req BindRequest) (ldap.LDAPRes // Get user info to store in context userInfo, _, err := fe.ApiClient().CoreApi.CoreUsersMeRetrieve(context.Background()).Execute() if err != nil { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "bind", + "reason": "user_info_fail", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() req.log.WithError(err).Warning("failed to get user info") return ldap.LDAPResultOperationsError, nil } diff --git a/internal/outpost/ldap/instance_search.go b/internal/outpost/ldap/instance_search.go index b18235f06..f6e89dee2 100644 --- a/internal/outpost/ldap/instance_search.go +++ b/internal/outpost/ldap/instance_search.go @@ -8,7 +8,10 @@ import ( "github.com/getsentry/sentry-go" "github.com/nmcclain/ldap" + "github.com/prometheus/client_golang/prometheus" "goauthentik.io/api" + "goauthentik.io/internal/outpost/ldap/metrics" + "goauthentik.io/internal/utils" ) func (pi *ProviderInstance) SearchMe(req SearchRequest, f UserFlags) (ldap.ServerSearchResult, error) { @@ -32,12 +35,30 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult, entries := []*ldap.Entry{} filterEntity, err := ldap.GetFilterObjectClass(req.Filter) if err != nil { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "search", + "reason": "filter_parse_fail", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultOperationsError}, fmt.Errorf("Search Error: error parsing filter: %s", req.Filter) } if len(req.BindDN) < 1 { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "search", + "reason": "empty_bind_dn", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultInsufficientAccessRights}, fmt.Errorf("Search Error: Anonymous BindDN not allowed %s", req.BindDN) } if !strings.HasSuffix(req.BindDN, baseDN) { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "search", + "reason": "invalid_bind_dn", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultInsufficientAccessRights}, fmt.Errorf("Search Error: BindDN %s not in our BaseDN %s", req.BindDN, pi.BaseDN) } @@ -46,6 +67,12 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult, pi.boundUsersMutex.RUnlock() if !ok { pi.log.Debug("User info not cached") + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "search", + "reason": "user_info_not_cached", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultInsufficientAccessRights}, errors.New("access denied") } if !flags.CanSearch { @@ -56,6 +83,12 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult, parsedFilter, err := ldap.CompileFilter(req.Filter) if err != nil { + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "search", + "reason": "filter_parse_fail", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultOperationsError}, fmt.Errorf("Search Error: error parsing filter: %s", req.Filter) } @@ -65,6 +98,12 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult, switch filterEntity { default: + metrics.RequestsRejected.With(prometheus.Labels{ + "type": "search", + "reason": "unhandled_filter_type", + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Inc() return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultOperationsError}, fmt.Errorf("Search Error: unhandled filter type: %s [%s]", filterEntity, req.Filter) case GroupObjectClass: wg := sync.WaitGroup{} diff --git a/internal/outpost/ldap/metrics/metrics.go b/internal/outpost/ldap/metrics/metrics.go new file mode 100644 index 000000000..6723f4a56 --- /dev/null +++ b/internal/outpost/ldap/metrics/metrics.go @@ -0,0 +1,33 @@ +package metrics + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + Requests = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "authentik_outpost_ldap_requests", + Help: "The total number of configured providers", + }, []string{"type", "dn", "filter", "client"}) + RequestsRejected = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "authentik_outpost_ldap_requests_rejected", + Help: "Total number of rejected requests", + }, []string{"type", "reason", "dn", "client"}) +) + +func RunServer() { + m := mux.NewRouter() + m.HandleFunc("/akprox/ping", func(rw http.ResponseWriter, r *http.Request) { + rw.WriteHeader(204) + }) + m.Path("/metrics").Handler(promhttp.Handler()) + err := http.ListenAndServe("0.0.0.0:9300", m) + if err != nil { + panic(err) + } +} diff --git a/internal/outpost/ldap/search.go b/internal/outpost/ldap/search.go index 34b634e5a..5f597c9f6 100644 --- a/internal/outpost/ldap/search.go +++ b/internal/outpost/ldap/search.go @@ -10,7 +10,9 @@ import ( goldap "github.com/go-ldap/ldap/v3" "github.com/google/uuid" "github.com/nmcclain/ldap" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "goauthentik.io/internal/outpost/ldap/metrics" "goauthentik.io/internal/utils" ) @@ -43,6 +45,12 @@ func (ls *LDAPServer) Search(bindDN string, searchReq ldap.SearchRequest, conn n defer func() { span.Finish() + metrics.Requests.With(prometheus.Labels{ + "type": "search", + "filter": req.Filter, + "dn": req.BindDN, + "client": utils.GetIP(req.conn.RemoteAddr()), + }).Observe(float64(span.EndTime.Sub(span.StartTime))) req.log.WithField("took-ms", span.EndTime.Sub(span.StartTime).Milliseconds()).Info("Search request") }() diff --git a/internal/outpost/proxyv2/metrics/metrics.go b/internal/outpost/proxyv2/metrics/metrics.go index 1d3e1d92c..585fb77da 100644 --- a/internal/outpost/proxyv2/metrics/metrics.go +++ b/internal/outpost/proxyv2/metrics/metrics.go @@ -22,8 +22,11 @@ var ( func RunServer() { m := mux.NewRouter() + m.HandleFunc("/akprox/ping", func(rw http.ResponseWriter, r *http.Request) { + rw.WriteHeader(204) + }) m.Path("/metrics").Handler(promhttp.Handler()) - err := http.ListenAndServe("localhost:9300", m) + err := http.ListenAndServe("0.0.0.0:9300", m) if err != nil { panic(err) } diff --git a/internal/outpost/proxyv2/proxyv2.go b/internal/outpost/proxyv2/proxyv2.go index 18ccdda52..fdc55e46b 100644 --- a/internal/outpost/proxyv2/proxyv2.go +++ b/internal/outpost/proxyv2/proxyv2.go @@ -63,7 +63,6 @@ func NewProxyServer(ac *ak.APIController, portOffset int) *ProxyServer { akAPI: ac, defaultCert: defaultCert, } - globalMux.Path("/akprox/ping").HandlerFunc(s.HandlePing) globalMux.PathPrefix("/akprox/static").HandlerFunc(s.HandleStatic) rootMux.PathPrefix("/").HandlerFunc(s.Handle) return s diff --git a/internal/web/metrics.go b/internal/web/metrics.go new file mode 100644 index 000000000..0d71197a1 --- /dev/null +++ b/internal/web/metrics.go @@ -0,0 +1,58 @@ +package web + +import ( + "io/ioutil" + "net/http" + + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "goauthentik.io/internal/config" +) + +var ( + Requests = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "authentik_main_requests", + Help: "The total number of configured providers", + }, []string{"dest"}) +) + +func RunMetricsServer() { + m := mux.NewRouter() + m.Path("/metrics").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + defer promhttp.InstrumentMetricHandler( + prometheus.DefaultRegisterer, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + DisableCompression: true, + }), + ).ServeHTTP(rw, r) + + // Get upstream metrics + re, err := http.NewRequest("GET", "http://localhost:8000/metrics/", nil) + if err != nil { + log.WithError(err).Warning("failed to get upstream metrics") + return + } + re.SetBasicAuth("monitor", config.G.SecretKey) + res, err := http.DefaultClient.Do(re) + if err != nil { + log.WithError(err).Warning("failed to get upstream metrics") + return + } + bm, err := ioutil.ReadAll(res.Body) + if err != nil { + log.WithError(err).Warning("failed to get upstream metrics") + return + } + _, err = rw.Write(bm) + if err != nil { + log.WithError(err).Warning("failed to get upstream metrics") + return + } + }) + err := http.ListenAndServe(config.G.Web.ListenMetrics, m) + if err != nil { + panic(err) + } +} diff --git a/internal/web/web_proxy.go b/internal/web/proxy.go similarity index 78% rename from internal/web/web_proxy.go rename to internal/web/proxy.go index d32d4e4b3..c33302894 100644 --- a/internal/web/web_proxy.go +++ b/internal/web/proxy.go @@ -5,7 +5,9 @@ import ( "net/http" "net/http/httputil" "net/url" + "time" + "github.com/prometheus/client_golang/prometheus" "goauthentik.io/internal/utils/web" ) @@ -28,18 +30,29 @@ func (ws *WebServer) configureProxy() { rp.ModifyResponse = ws.proxyModifyResponse ws.m.PathPrefix("/akprox").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { if ws.ProxyServer != nil { + before := time.Now() ws.ProxyServer.Handle(rw, r) + Requests.With(prometheus.Labels{ + "dest": "embedded_outpost", + }).Observe(float64(time.Since(before))) return } ws.proxyErrorHandler(rw, r, fmt.Errorf("proxy not running")) }) ws.m.PathPrefix("/").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { host := web.GetHost(r) + before := time.Now() if ws.ProxyServer != nil { if ws.ProxyServer.HandleHost(host, rw, r) { + Requests.With(prometheus.Labels{ + "dest": "embedded_outpost", + }).Observe(float64(time.Since(before))) return } } + Requests.With(prometheus.Labels{ + "dest": "py", + }).Observe(float64(time.Since(before))) ws.log.WithField("host", host).Trace("routing to application server") rp.ServeHTTP(rw, r) }) diff --git a/internal/web/web_ssl.go b/internal/web/ssl.go similarity index 100% rename from internal/web/web_ssl.go rename to internal/web/ssl.go diff --git a/internal/web/web_static.go b/internal/web/static.go similarity index 100% rename from internal/web/web_static.go rename to internal/web/static.go diff --git a/internal/web/web_static_utils.go b/internal/web/static_utils.go similarity index 100% rename from internal/web/web_static_utils.go rename to internal/web/static_utils.go diff --git a/internal/web/web_utils.go b/internal/web/utils.go similarity index 100% rename from internal/web/web_utils.go rename to internal/web/utils.go diff --git a/ldap.Dockerfile b/ldap.Dockerfile index 3cb70964f..040400767 100644 --- a/ldap.Dockerfile +++ b/ldap.Dockerfile @@ -31,6 +31,6 @@ ENV GIT_BUILD_HASH=$GIT_BUILD_HASH COPY --from=builder /go/ldap / -HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9000/akprox/ping" ] +HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9300/akprox/ping" ] ENTRYPOINT ["/ldap"] diff --git a/proxy.Dockerfile b/proxy.Dockerfile index 2ff255698..702cf4d6b 100644 --- a/proxy.Dockerfile +++ b/proxy.Dockerfile @@ -43,6 +43,6 @@ ENV GIT_BUILD_HASH=$GIT_BUILD_HASH COPY --from=builder /go/proxy / -HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9000/akprox/ping" ] +HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9300/akprox/ping" ] ENTRYPOINT ["/proxy"] diff --git a/website/docs/outposts/outposts.md b/website/docs/outposts/outposts.md index f05dabff7..1636076ce 100644 --- a/website/docs/outposts/outposts.md +++ b/website/docs/outposts/outposts.md @@ -61,6 +61,7 @@ kubernetes_service_type: ClusterIP # - 'secret' # - 'deployment' # - 'service' +# - 'prometheus servicemonitor' # - 'ingress' # - 'traefik middleware' kubernetes_disabled_components: []