diff --git a/authentik/sources/ldap/models.py b/authentik/sources/ldap/models.py
index a09791593..2b939e022 100644
--- a/authentik/sources/ldap/models.py
+++ b/authentik/sources/ldap/models.py
@@ -1,5 +1,7 @@
 """authentik LDAP Models"""
 from os import chmod
+from os.path import dirname, exists
+from shutil import rmtree
 from ssl import CERT_REQUIRED
 from tempfile import NamedTemporaryFile, mkdtemp
 from typing import Optional
@@ -189,6 +191,11 @@ class LDAPSource(Source):
                 raise exc
             server_kwargs["get_info"] = NONE
             return self.connection(server, server_kwargs, connection_kwargs)
+        finally:
+            if connection.server.tls.certificate_file is not None and exists(
+                connection.server.tls.certificate_file
+            ):
+                rmtree(dirname(connection.server.tls.certificate_file))
         return RuntimeError("Failed to bind")
 
     @property