diff --git a/Dockerfile b/Dockerfile
index ece04b6e0..9c6a8e35b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -93,8 +93,8 @@ RUN apt-get update && \
     rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/ && \
     adduser --system --no-create-home --uid 1000 --group --home /authentik authentik && \
     mkdir -p /certs /media /blueprints && \
-    mkdir -p /authentik/.ssh && \
-    chown authentik:authentik /certs /media /authentik/.ssh
+    chown authentik:authentik /certs /media /authentik/ && \
+    chgrp authentik /etc/ssh/ssh_config.d
 
 COPY ./authentik/ /authentik
 COPY ./pyproject.toml /
diff --git a/authentik/outposts/docker_ssh.py b/authentik/outposts/docker_ssh.py
index b146ca6c9..3dfa626ee 100644
--- a/authentik/outposts/docker_ssh.py
+++ b/authentik/outposts/docker_ssh.py
@@ -9,7 +9,7 @@ from authentik.crypto.models import CertificateKeyPair
 
 HEADER = "### Managed by authentik"
 FOOTER = "### End Managed by authentik"
-
+SSH_CONFIG_DIR = Path("/etc/ssh/ssh_config.d/")
 
 def opener(path, flags):
     """File opener to create files as 700 perms"""
@@ -33,8 +33,8 @@ class DockerInlineSSH:
     def __init__(self, host: str, keypair: CertificateKeyPair) -> None:
         self.host = host
         self.keypair = keypair
-        self.config_path = Path("~/.ssh/config").expanduser()
-        if self.config_path.exists() and HEADER not in self.config_path.read_text(encoding="utf-8"):
+        self.config_path = SSH_CONFIG_DIR / Path(self.host + ".conf")
+        if self.config_path.exists():
             # SSH Config file already exists and there's no header from us, meaning that it's
             # been externally mapped into the container for more complex configs
             raise SSHManagedExternallyException(
@@ -42,21 +42,16 @@ class DockerInlineSSH:
             )
         if not self.keypair:
             raise DockerException("keypair must be set for SSH connections")
-        self.header = f"{HEADER} - {self.host}\n"
 
     def write_config(self, key_path: str) -> bool:
         """Update the local user's ssh config file"""
-        with open(self.config_path, "a+", encoding="utf-8") as ssh_config:
-            if self.header in ssh_config.readlines():
-                return False
+        with open(self.config_path, "w", encoding="utf-8") as ssh_config:
             ssh_config.writelines(
                 [
-                    self.header,
                     f"Host {self.host}\n",
                     f"    IdentityFile {key_path}\n",
                     "    StrictHostKeyChecking No\n",
                     "    UserKnownHostsFile /dev/null\n",
-                    f"{FOOTER}\n",
                     "\n",
                 ]
             )
@@ -72,26 +67,16 @@ class DockerInlineSSH:
     def write(self):
         """Write keyfile and update ssh config"""
         self.key_path = self.write_key()
-        was_written = self.write_config(self.key_path)
-        if not was_written:
+        try:
+            self.write_config(self.key_path)
+        except OSError:
             self.cleanup()
 
     def cleanup(self):
         """Cleanup when we're done"""
         try:
             os.unlink(self.key_path)
-            with open(self.config_path, "r", encoding="utf-8") as ssh_config:
-                start = 0
-                end = 0
-                lines = ssh_config.readlines()
-                for idx, line in enumerate(lines):
-                    if line == self.header:
-                        start = idx
-                    if start != 0 and line == f"{FOOTER}\n":
-                        end = idx
-            with open(self.config_path, "w+", encoding="utf-8") as ssh_config:
-                lines = lines[:start] + lines[end + 2 :]
-                ssh_config.writelines(lines)
+            os.unlink(self.config_path)
         except OSError:
             # If we fail deleting a file it doesn't matter that much
             # since we're just in a container